robots.txt 760 Bytes
Newer Older
Sam Hocevar's avatar
Sam Hocevar committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26
# $Id$

User-agent: *
Disallow: /pub
Disallow: /removed

# Do not crawl CVS and .svn directories
User-agent: *
Disallow: CVS
Disallow: .svn

# "This robot collects content from the Internet for the sole purpose of
# helping educational institutions prevent plagiarism. [...] we compare
# student papers against the content we find on the Internet to see if we
# can find similarities." (http://www.turnitin.com/robot/crawlerinfo.html)
#  --> fuck off.
User-Agent: TurnitinBot
Disallow: /

# "NameProtect engages in crawling activity in search of a wide range of
# brand and other intellectual property violations that may be of interest
# to our clients." (http://www.nameprotect.com/botinfo.html)
#  --> fuck off.
User-Agent: NPBot
Disallow: /

Christophe Mutricy's avatar
Christophe Mutricy committed
27
# "iThenticate