# allows these search robots
User-agent: Googlebot
Disallow: 

User-agent: MSNBot
Disallow: 

User-agent: Slurp
Disallow: 

User-agent: Teoma
Disallow: 

User-agent: Gigabot
Disallow: 

User-agent: Scrubby
Disallow: 

User-agent: Robozilla
Disallow: 

User-agent: ia_archiver
Disallow: 

User-agent: naverbot
Disallow: 

User-agent: yeti
Disallow: 


# disallows remaining spiders
User-agent: *
Disallow: /


# delays crawlers
Crawl-delay: 15


# disallows various directories
Disallow: /cgi-bin/
Disallow: /xtra/
Disallow: /gallery/
Disallow: /galleries/


# hides various WordPress sections
Disallow: /wp-
Disallow: /feed/
Disallow: /trackback/
Disallow: /rss/
Disallow: /comments/feed/
Disallow: /page/
Disallow: /date/
Disallow: /comments/
Disallow: /cgi-bin/
Disallow: /2005/
Disallow: /2006/
Disallow: /2007/
Disallow: /2008/
Disallow: /2009/
Disallow: /2010/
Disallow: /2011/
Disallow: /2012/
Disallow: /*?*
Disallow: /iframes/
Disallow: /recommends/


# disallows various crawler specific activities
User-agent: Googlebot-Image
Disallow: /*.PDF$
Disallow: /*.MP3$
Disallow: /*.MP4$

User-agent: msnbot
Disallow: /*.PDF$
Disallow: /*.MP3$
Disallow: /*.MP4$