# config for _all_ crawlers
# please keep in alphabetic order so it's easy to find things
User-agent: *

Crawl-delay: 1

Disallow: /admin
Disallow: /api
Disallow: /cms
Disallow: /ping
Disallow: /sales
Disallow: /user
Disallow: /_gone

Sitemap: https://www.sciencephoto.com/sitemap.xml

User-agent: Arquivo-web-crawler
Disallow: /lightbox/
Disallow: /ping/
Disallow: /public/basket/

# Stub for bravebot, so that it identifies itself via HTTP requests
User-agent: bravebot
Disallow: /api

User-Agent: FacebookBot
Disallow: /...

User-agent: adbeat_bot
Disallow: /

User-agent: adsbot
Disallow: /

User-agent: AHC
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: AiHitBot
Disallow: /

User-agent: aiohttp
Disallow: /

User-agent: AwarioRssBot
User-agent: AwarioSmartBot
Disallow: /

User-agent: Barkrowler
Disallow: /

User-agent: Bingbot
Disallow: /accounts
Disallow: /search

User-agent: BLEXBot
Disallow: /

User-agent: Buck
Disallow: /

User-agent: Cincraw
Disallow: /

User-agent: crystalsemantics
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent : DAUM
Disallow : /

User-agent: DomCopBot
Disallow: /

User-agent: dotbot
Disallow: /

User-agent: Go-http-client
Disallow: /

User-agent: grapeshot
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: Linespider
Disallow: /keywords/
Disallow: /ping

User-agent: ltx71 - (http://ltx71.com/)
Disallow: /

User-agent: Mail.RU_Bot
Disallow: /

User-agent: MauiBot
Disallow: /

User-agent: MegaIndex
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: MTRobot
Disallow: /

User-Agent: Neevabot
Disallow: /%20%28PROTECTED%29

User-agent: NetpeakCheckerBot
Disallow: /

User-agent: PetalBot
Disallow: /art/
Disallow: /hashtag/
Disallow: /search

User-Agent: panscient.com
Disallow: /

User-agent: proximic
Disallow: /

User-agent: Riddler
Disallow: /

User-agent: rogerbot
Disallow: /

User-agent: Scrapy
Disallow: /

User-agent: Screaming Frog SEO Spider
Disallow: /

User-agent: Semanticbot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: serpstatbot
Disallow: /

User-agent: Screaming Frog SEO Spider
Disallow: /

User-agent: SEOkicks
Disallow: /

User-agent: SiteCheckerBotCrawler
Disallow: /

User-agent: StormCrawler
Disallow: /

User-agent: The Knowledge AI
Disallow: /

User-Agent: trendictionbot
Disallow: /

User-agent: Twingly Recon
Disallow: /

User-agent: VelenPublicWebCrawler
Disallow: /

User-Agent: WellKnownBot
Disallow: /

User-agent: WikiDo
Disallow: /

User-agent: XoviBot
Disallow: /

User-agent: YaK
Disallow: /

User-agent: ZoominfoBot
Disallow: /