Sitemap: http://projectcensored.org/sitemap.xml.gz

User-agent: *
Disallow: /cgi-bin/
Disallow: /wp-admin/
Disallow: /wp-content/
Disallow: /wp-includes/
User-agent: Googlebot
Disallow: /*.php$
Disallow: /*.js$
Disallow: /*.inc$
Disallow: /*.css$
User-agent: beetween/Nutch-1.0 (beetween crawler)
Disallow: /
User-agent: beetween/Nutch-1.0
Disallow: /
User-agent: DataPatrol/Nutch-1.0 (DataPatrol)
Disallow: /
User-agent: SBIder/Nutch-1.0-dev (http://www.sitesell.com/sbider.html)
Disallow: /
User-agent: Infoaxe./Nutch-0.9
Disallow: /
User-agent: PiplBot; http://pipl.com/bot/
Disallow: /
User-agent: CamelHttpStream/1.0 Evolution/2.26.1
Disallow: /
User-agent: Exabot
Disallow: /
User-agent: Yandex/1.01.001 (spider64.yandex.ru)
Disallow: /
User-agent: Yandex (spider64.yandex.ru)
Disallow: /
User-agent: YandexSomething/1.0 (spider64.yandex.ru)
Disallow: /
User-agent: Yandex/1.01.001 (sticker00.yandex.ru)