#
# robots.txt for peptidecritic.com
#
# TERMS OF USE NOTICE
# Automated access to this site is governed by our Terms and Conditions:
#   https://peptidecritic.com/terms
# See Section 7A (Ratings and Compiled Data) and Section 7B (Automated Access).
#
# Our vendor ratings, review counts, price data, verification flags, and
# rankings are a compiled data set. Systematic collection, redistribution,
# or reuse in another index, score, rating, comparison tool, or training
# corpus requires a written license. Limited editorial quotation, and a
# listed vendor's display of its own current rating, are permitted under
# Terms Section 7A.3. Commercial licensing enquiries:
#   https://peptidecritic.com/contact
#
# Public visibility of a page does not itself grant permission for
# automated collection or commercial reuse beyond the permissions in our
# Terms. Assistive technologies and ordinary browser functions are not
# restricted. If we notify you that your access is prohibited, you must
# stop and must not evade the restriction (Terms Section 7B.4).
#
# Public search engines may crawl to build a general-purpose public search
# index, subject to the rules below. Crawling to train machine-learning
# models, to build a competing product or data set, or to extract our
# compiled data is not permitted.
#

# ---------------------------------------------------------------------------
# AI / LLM training and data-harvesting crawlers - disallowed in full.
# Permitting these would contradict Terms Section 7A.2(b).
# ---------------------------------------------------------------------------
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: CCBot
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: anthropic-ai
User-agent: Google-Extended
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Applebot-Extended
User-agent: Bytespider
User-agent: Amazonbot
User-agent: Meta-ExternalAgent
User-agent: meta-externalagent
User-agent: FacebookBot
User-agent: cohere-ai
User-agent: Diffbot
User-agent: Omgilibot
User-agent: Timpibot
User-agent: ImagesiftBot
User-agent: PanguBot
User-agent: YouBot
User-agent: AI2Bot
Disallow: /

# ---------------------------------------------------------------------------
# SEO / competitive-intelligence scrapers - disallowed in full.
# These exist to resell our compiled data.
# ---------------------------------------------------------------------------
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: DataForSeoBot
User-agent: BLEXBot
User-agent: SerpstatBot
User-agent: MegaIndex
User-agent: ZoominfoBot
User-agent: Barkrowler
User-agent: PetalBot
User-agent: SeekportBot
Disallow: /

# ---------------------------------------------------------------------------
# All other crawlers, including public search engines.
# ---------------------------------------------------------------------------
User-agent: *
# CSS, JS, Images
Allow: /core/*.css$
Allow: /core/*.css?
Allow: /core/*.js$
Allow: /core/*.js?
Allow: /core/*.gif
Allow: /core/*.jpg
Allow: /core/*.jpeg
Allow: /core/*.png
Allow: /core/*.svg
Allow: /profiles/*.css$
Allow: /profiles/*.css?
Allow: /profiles/*.js$
Allow: /profiles/*.js?
Allow: /profiles/*.gif
Allow: /profiles/*.jpg
Allow: /profiles/*.jpeg
Allow: /profiles/*.png
Allow: /profiles/*.svg
# Directories
Disallow: /core/
Disallow: /profiles/
# Files
Disallow: /README.md
Disallow: /composer/Metapackage/README.txt
Disallow: /composer/Plugin/ProjectMessage/README.md
Disallow: /composer/Plugin/Scaffold/README.md
Disallow: /composer/Plugin/VendorHardening/README.txt
Disallow: /composer/Template/README.txt
Disallow: /modules/README.txt
Disallow: /sites/README.txt
Disallow: /themes/README.txt
Disallow: /web.config
# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register
Disallow: /user/password
Disallow: /user/login
Disallow: /user/logout
Disallow: /media/oembed
Disallow: /*/media/oembed
# Paths (no clean URLs)
Disallow: /index.php/admin/
Disallow: /index.php/comment/reply/
Disallow: /index.php/filter/tips
Disallow: /index.php/node/add/
Disallow: /index.php/search/
Disallow: /index.php/user/password
Disallow: /index.php/user/register
Disallow: /index.php/user/login
Disallow: /index.php/user/logout
Disallow: /index.php/media/oembed
Disallow: /index.php/*/media/oembed

# Sitemap
Sitemap: https://peptidecritic.com/sitemap.xml

# Peptide Critic additions
# /go/ = affiliate redirect endpoints; no crawl value, and crawling them
# fires outbound vendor requests.
Disallow: /go/
Disallow: /api/
Disallow: /search
Disallow: /*?cb=
Disallow: /*?nocache=

Sitemap: https://peptidecritic.com/sitemap.xml