# CrawfordNote:
# Exclude e-Xoops directories not suitable for spidering. 
User-agent: *
Disallow: /cache/
Disallow: /admin/
Disallow: /class/
Disallow: /include/
Disallow: /themes/
Disallow: /images/
Disallow: /language/
Disallow: /system/
Disallow: /manual/
# exclude help system from robots
Disallow: /manual/ 
Disallow: /doc/ 
Disallow: /gif/
# disallow the messaging page
Disallow: /message/
# but allow htdig to index our doc-tree
User-agent: susedig
Disallow:
# disallow stress test
user-agent: stress-agent
Disallow: /