# /robots.txt file for http://webcrawler.com/
# mail webmaster@webcrawler.com for constructive criticism

User-agent: *
Crawl-Delay: 10
Disallow: /tmp
Disallow: /logs
Disallow: /media/aggregator/
Disallow: /media/tracker/
Disallow: /media/comment/reply
Disallow: /media/node/add/
Disallow: /media/user/
Disallow: /media/search/
Disallow: /media/book/export/
Disallow: /media/book/print/
Disallow: /media/admin/
Disallow: /media/forward/
Disallow: /media/schedule/
Disallow: /media/og/manage/