# http://www.robotstxt.org # https://www.google.com/support/webmasters/bin/answer.py?hl=en&answer=156449 # Point crawlers to sitemap which allows more specific crawling Sitemap: https://shopforce-platform.herokuapp.com/sitemap/ # Indicate a delay in seconds in between requests to slow down bots, note that GoogleBot ignores this directive User-agent: * Crawl-delay: 30 # Disallow nothing for all UserAgents User-agent: * Disallow: # https://ahrefs.com/robot User-agent: AhrefsBot Disallow: / # https://www.aihitdata.com/about User-agent: aiHitBot Disallow: / # http://www.backlinktest.com/crawler.html User-agent: BacklinkCrawler Disallow: / # http://www.brandwatch.com/magpie-crawler/ User-agent: magpie-crawler Disallow: / # http://www.career-x.de/bot.html User-agent: careerbot Disallow: / # http://www.cloudservermarket.com User-Agent: CloudServerMarketSpider Disallow: / # http://crawler.sistrix.net/ User-agent: sistrix User-agent: SISTRIX Crawler User-agent: SISTRIX Disallow: / # http://www.datenbutler.de User-agent: mindUpBot Disallow: / # http://www.domaintools.com/webmasters/surveybot.php User-agent: SurveyBot Disallow: / # http://www.exalead.com User-agent: Exabot Disallow: / # Facebook crawls very aggressively # Throttling using Cloudflare RateLimitRules is challenging, since these rules are "per-domain," but a lot of domains run on the same server. # Logs indicated that dozens of requests to different domains were issued per second, but the RateLimit did not activate... # https://developers.facebook.com/docs/sharing/webmasters/crawler User-agent: meta-externalads User-agent: meta-externalagent User-agent: meta-webindexer Disallow: / # http://filterdb.iss.net/crawler/ User-agent: oBot Disallow: / User-agent: fr-crawler Disallow: / # http://jooble.org/jooble-bot User-agent: JoobleBot Disallow: / # https://www.jobs.de/s/web-crawler User-agent: jobs.de-Robot Disallow: / # https://www.linkdex.com/en-us/about/bots/ User-agent: linkdexbot Disallow: / User-agent: MauiBot Disallow: / # http://www.majestic12.co.uk/projects/dsearch/mj12bot.php User-agent: MJ12bot Disallow: / # http://www.meanpath.com/meanpathbot.html User-agent: meanpathbot Disallow: / # https://megaindex.com/crawler User-agent: MegaIndex.ru User-agent: megaindex.com Disallow: / User-agent: MetaJobBot Disallow: / # https://aspiegel.com/petalbot # https://webmaster.petalsearch.com/site/petalbot User-agent: PetalBot Disallow: / # http://openlinkprofiler.org/bot User-agent: spbot Disallow: / # http://www.opensiteexplorer.org/dotbot # http://moz.com/researchtools/ose/dotbot User-agent: dotbot Disallow: / # http://www.searchmetrics.com User-agent: SearchmetricsBot Disallow: / # https://www.seokicks.de/robot.html User-agent: SEOkicks-Robot Disallow: / # http://www.seodiver.com/bot User-agent: SEOdiver Disallow: / User-agent: seoscanners.net Disallow: / # Bot der Leipziger Unister Holding GmbH User-agent: UnisterBot Disallow: / # http://www.wotbox.com/bot/ User-agent: wotbox Disallow: / # http://webmeup-crawler.com User-agent: BLEXBot Disallow: / # http://www.trendiction.de/de/publisher/bot User-Agent: trendictionbot Disallow: / # https://www.lipperhey.com/en/about/ User-agent: Lipperhey-Kaus-Australis Disallow: / User-agent: Spiderbot Disallow: / User-agent: LinkStats Disallow: / User-agent: JobboerseBot Disallow: / User-agent: ICCrawler Disallow: / User-agent: Plista Disallow: / User-agent: Domain Re-Animator Bot Disallow: / # https://www.lipperhey.com/en/about/ User-agent: Lipperhey-Kaus-Australis Disallow: / # https://turnitin.com/robot/crawlerinfo.html User-agent: turnitinbot Disallow: / # http://help.coccoc.com/ User-agent: coccoc Disallow: / # http://ubermetrics-technologies.com User-agent: um-IC Disallow: / # http://searchgears.de/uber-uns/crawling-faq.html User-agent: sg-Orbiter Disallow: / # http://commoncrawl.org/faq/ User-agent: CCBot Disallow: / # https://www.qwant.com/ User-agent: Qwantify Disallow: / # http://linkfluence.net/ User-agent: Kraken Disallow: / # http://www.botje.com/plukkie.htm User-agent: plukkie Disallow: / # https://www.safedns.com/searchbot User-agent: SafeDNSBot Disallow: / # http://www.haosou.com/help/help_3_2.html User-agent: 360Spider Disallow: / # http://www.haosou.com/help/help_3_2.html User-agent: HaosouSpider Disallow: / # http://www.moz.com/dp/rogerbot User-agent: rogerbot Disallow: / # http://www.openhose.org/bot.html User-agent: OpenHoseBot Disallow: / # http://www.screamingfrog.co.uk/seo-spider/ User-agent: Screaming Frog SEO Spider Disallow: / # http://thumbsniper.com User-agent: ThumbSniper Disallow: / # http://www.radian6.com/crawler User-agent: R6_CommentReader Disallow: / User-agent: ImplisenseBot Disallow: / # http://cliqz.com/company/cliqzbot User-agent: Cliqzbot Disallow: / # http://www.trendiction.com/en/publisher/bot User-Agent: trendictionbot Disallow: / # http://warebay.com/bot.html User-agent: WBSearchBot Disallow: / # http://yandex.com/bots User-agent: Yandex Disallow: / # https://www.semrush.com/bot/ User-agent: SemrushBot Disallow: / User-agent: SemrushBot-SA Disallow: /