# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file # # To ban all spiders from the entire site uncomment the next two lines: Sitemap: https://newstral.com/sitemap_index.xml User-agent: * Disallow: /nl/maps Disallow: /nl/regions Disallow: /nl/people Disallow: /nl/organisations Disallow: /en/cars Disallow: /es/cars Disallow: /nl/cars Disallow: /de/article/en Disallow: /de/article/es Disallow: /en/article/de Disallow: /en/article/es Disallow: /en/article/nl Disallow: /es/article/en Disallow: /es/article/de Disallow: /es/article/nl Disallow: /nl/article/en Disallow: /nl/article/es Disallow: /nl/article/de Disallow: /sources/1 Disallow: /sources/2 Disallow: /sources/3 Disallow: /sources/4 Disallow: /sources/5 Disallow: /sources/6 Disallow: /sources/7 Disallow: /sources/8 Disallow: /sources/9 User-agent: msnbot Crawl-delay: 10 User-agent: TurnitinBot Disallow: / User-agent: Seobility Disallow: / User-agent: James BOT Disallow: / User-agent: SEOstats Disallow: / User-agent: Baiduspider Disallow: / User-agent: waybackarchive.org Disallow: / User-agent: EasouSpider Disallow: / User-agent: YoudaoBot Disallow: / User-agent: 360Spider Disallow: / User-agent: YisouSpider Disallow: / User-agent: http://www.baidu.com/search/spider.html Disallow: / User-agent: Sogou web spider Disallow: / User-agent: proximic Disallow: / User-agent: YandexBot Disallow: / User-agent: YandexImages Disallow: / User-agent: Ezooms Disallow: / User-agent: UnisterBot Disallow: / User-agent: KatBot Disallow: / User-agent: MJ12bot Disallow: / User-agent: archive.org_bot Disallow: / User-agent: Exabot Disallow: / User-agent: AhrefsBot Disallow: / User-agent: CrystalSemanticsBot Disallow: / User-agent: fastbot Disallow: / User-agent: Mail.RU_Bot Disallow: / User-agent: meanpathbot Disallow: / User-agent: Yeti Disallow: / User-agent: sSearch Disallow: / User-agent: SurveyBot Disallow: / User-agent: spbot Disallow: / User-agent: CareerBot Disallow: / User-agent: BLEXbot Disallow: / User-agent: niki-bot Disallow: / User-agent: bot-pge.chlooe.com Disallow: / User-agent: SISTRIX Disallow: / User-agent: GrapeshotCrawler Disallow: / User-agent: CMS Crawler Disallow: / User-agent: SearchmetricsBot Disallow: /