# scrapy.org
# by default ROBOTSTXT_OBEY is set to false 
#	http://doc.scrapy.org/en/latest/topics/settings.html
User-agent: Scrapy
User-agent: MJ12bot
User-agent: serpstatbot
User-agent: AhrefsBot
Disallow: /