User-agent: *
Crawl-delay: 5 # задает таймаут в 5 секунды 

User-agent: Twiceler
Disallow: /

User-agent: Slurp
Disallow: /

User-agent: msnbot-media
Disallow: /

User-agent: msnbot-news
Disallow: /

User-agent: msnbot-products
Disallow: /

User-agent: msnbot
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: Liveinternet
Disallow: /