# robots.txt!

User-agent: *

# Less stuff!
Crawl-Delay: 1
Request-rate: 30/1m             # one document every 2 seconds

Disallow: /jvs     # testing version of jbovlaste, no reason to index.

Disallow: /jbovlaste/natlang # the really big, non-static parts of
Disallow: /jbovlaste/dict    # jbovlaste. the rest of it ought to
                             # be OK (and in fact, good, to expose).

Disallow: /phpbb
Disallow: /bpfk

Disallow: /irc

Disallow: /cgi-bin/aegis.cgi
Disallow: /cgi-bin/htsearch
Disallow: /cgi-bin/info2www
Disallow: /cgi-bin/lconline/lc.cgi
Disallow: /cgi-bin/listargate.cgi
Disallow: /cgi-bin/lsg2.cgi
Disallow: /cgi-bin/mserv.cgi
Disallow: /cgi-bin/mserv_main.cgi
Disallow: /cgi-bin/mserv_margin.cgi
Disallow: /cgi-bin/viewcvs-query.cgi
Disallow: /cgi-bin/viewcvs.cgi

Disallow: /lsg2
Disallow: /bugzilla
Disallow: /rt

Disallow: /htdig
Disallow: /story
Disallow: /lists
Disallow: /cvsweb
Disallow: /viewcvs

# Disallow all of the non-"static" parts of twiki
Disallow: /twiki/bin/attach
Disallow: /twiki/bin/changes
Disallow: /twiki/bin/edit
Disallow: /twiki/bin/geturl
Disallow: /twiki/bin/installpasswd
Disallow: /twiki/bin/mailnotify
Disallow: /twiki/bin/manage
Disallow: /twiki/bin/oops
Disallow: /twiki/bin/passwd
Disallow: /twiki/bin/poll
Disallow: /twiki/bin/preview
Disallow: /twiki/bin/rdiff
Disallow: /twiki/bin/register
Disallow: /twiki/bin/rename
Disallow: /twiki/bin/save
Disallow: /twiki/bin/search
Disallow: /twiki/bin/statistics
Disallow: /twiki/bin/testenv
Disallow: /twiki/bin/upload

# No spidering of Robin's media.
Disallow: /~rlpowell/media/

# No spidering of APC14
Disallow: /~raladue/apc14