User-agent: AhrefsBot
Disallow: /

#Baiduspider
User-agent: Baiduspider
Disallow: /

#Yandex
User-agent: Yandex
Disallow: /

#Sogou web spider
User-agent: Sogou
Disallow: /

#https://mj12bot.com/
User-agent: MJ12bot
Disallow: /

#https://www.exalead.com/search/webmasterguide
User-agent: Exabot
Disallow: /

#Alexa https://support.alexa.com/hc/en-us/articles/200450194-Alexa-s-Web-and-Site-Audit-Crawlers
User-agent: ia_archiver
Disallow: /

#http://phpcrawl.cuab.de/
User-agent: phpcrawl
Disallow: /

#BIGLOTRON (Beta 2;GNU/Linux)
User-agent: BIGLOTRON
Disallow: /

##the following list comes from timeanddate.com/robots.txt
User-agent: MSIECrawler
Disallow: /

User-agent: PortalBSpider
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Yahoo Pipes 1.0
Disallow: /

# disallow any urls with ? in
User-Agent: AhrefsBot
Disallow: /

User-Agent: dotbot
Disallow: /

User-agent: tadcrawler
Disallow: /

User-agent: ScoutJet
Disallow: /