# web/ak/robots.txt   2010-7-19   Alan U. Kennington.
# $Id: robots.txt 29119 2010-07-19 01:48:58Z akenning $
# robots.txt file for www.alankennington.com.

# The Altavista Scooter makes daily indexes, which is too often.
User-agent: Baiduspider
User-agent: dotbot
User-agent: ia_archiver
User-agent: panscient.com
User-agent: Scooter
User-agent: slurp
User-agent: Sogou
User-agent: sogou spider
User-agent: TencentTraveler
User-agent: Yandex
Disallow: /

# Inktomi uses "slurp". So does Yahoo. But they give me almost no hits.
User-agent: Googlebot
Disallow: /db/books/
Disallow: /extra/
Disallow: /human/
Disallow: /images/
Disallow: /php/
Disallow: /x/
Disallow: /z/
Disallow: /*.akm$
Disallow: /*.asc$
Disallow: /*.bz2$
Disallow: /*.c$
Disallow: /*.css$
Disallow: /*.csv$
Disallow: /*.dat$
Disallow: /*.doc$
Disallow: /*.gif$
Disallow: /*.gz$
Disallow: /*.h$
Disallow: /*.java$
Disallow: /*.jpg$
Disallow: /*.ly$
Disallow: /*.m4$
Disallow: /*.mid$
Disallow: /*.mp3$
Disallow: /*.ogg$
Disallow: /*.pdf$
Disallow: /*.pdf.gz$
Disallow: /*.phpmod$
Disallow: /*.pl$
Disallow: /*.pm$
Disallow: /*.png$
Disallow: /*.ps$
Disallow: /*.ps.gz$
Disallow: /*.ps.zip$
Disallow: /*.sys$
Disallow: /*.tex$
Disallow: /*.tgz$
Disallow: /*.tsv$
Disallow: /*.xml$
Disallow: /*.zip$
Disallow: /*.Z$
Disallow: /*/makefile
Disallow: /*/LICENCE
Disallow: /*/CHANGES

# Inktomi uses "slurp". So does Yahoo. But they give me almost no hits.
User-agent: msnbot
User-agent: Teoma
User-agent: twiceler
Disallow: /db/books/
Disallow: /extra/
Disallow: /human/
Disallow: /images/
Disallow: /php/
Disallow: /x/
Disallow: /tex/
Disallow: /z/

User-agent: *
Disallow: /