User-agent:*
Disallow: .pdf
Disallow: .doc
Disallow: .zip

User-Agent: Googlebot
Allow: /googlenews/

User-agent:gigabot
Disallow: /

User-agent:*WebReaper*
Disallow: /

User-agent:Teleport*
Disallow: /

User-agent:WebZIP*
Disallow: /

User-agent:WebMirror*
Disallow: /

User-agent:Offline Explorer*
Disallow: /

User-agent:ia_archiver*
Disallow: /

User-agent:Bloodhound*
Disallow: /

User-agent:Deweb*
Disallow: /

User-agent:SpiderBot*
Disallow: /

User-agent:PackRat*
Disallow: /

User-agent:TLSpider*
Disallow: /

User-agent:*Openbot/*
Disallow: /

User-agent:Googlebot-Image*
Disallow: /

User-agent:AbachoBOT*
Disallow: /

User-agent:EmailCollector*
Disallow: /

User-agent:MyCrawler*
Disallow: /

User-agent:Steeler*
Disallow: /

User-agent:*Vagabondo*
Disallow: /

User-agent:*QuepasaCreep*
Disallow: /

User-agent:larbin*
Disallow: /

User-agent:larbin_2*
Disallow: /

User-agent:NPBot
Disallow: /

User-agent: searchpreview
Disallow: /