# description at http://www.robotstxt.org/wc/exclusion-admin.html
# exclude internal stuff
User-agent: *
Disallow: /foobar/
Disallow: /intern/
Disallow: /petplay/
Disallow: /mailto.html
Disallow: /mailto/
Disallow: /novafox/

# no idea to what agent string msn reacts, if at all. but msn does not spread its requests over time, but bulk-gets everything - not nice at all
User-agent: msnbot
Disallow: /