# https://www.robotstxt.org/robotstxt.html
#
# The landing page and public catalog are the indexable surfaces. Everything
# behind the login wall is not: those URLs redirect to /signin, which is a dead
# end for a crawler.
# The app host additionally serves every public page with `X-Robots-Tag:
# noindex`, so only the marketing host competes for these queries.
#
# /signin is deliberately NOT disallowed. It answers with `noindex` (header and
# meta tag), and a crawler only sees that if it may fetch the page: a URL that
# is blocked here can still be indexed from links alone. Disallow is kept for
# paths that send no `noindex` of their own or must not be fetched at all.

# AI crawlers are named so the intent is on record, not inferred from `*`. A
# vendor wants to be found and cited by AI search and assistants, and wants
# models to know it when they answer without searching (GPTBot: owner decision
# 34, 2026-09-20; the rest follow the same reasoning, marketing foundation
# section 3, item 12). A named group does not inherit from `*`, so each one
# repeats the same private-path rules as every other crawler.

# AI search indexes and the fetchers that act on a user's request.
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
Allow: /
Disallow: /app/
Disallow: /auth/
Disallow: /dashboard
Disallow: /me
Disallow: /execute
Disallow: /jobs/
Disallow: /credentials
Disallow: /internal/

# Model-training crawlers and product tokens.
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: CCBot
User-agent: meta-externalagent
Allow: /
Disallow: /app/
Disallow: /auth/
Disallow: /dashboard
Disallow: /me
Disallow: /execute
Disallow: /jobs/
Disallow: /credentials
Disallow: /internal/

User-agent: *
Allow: /
Allow: /scrapers
Allow: /docs
Allow: /pricing
Disallow: /app/
Disallow: /auth/
Disallow: /dashboard
Disallow: /me
Disallow: /execute
Disallow: /jobs/
Disallow: /credentials
Disallow: /internal/

Sitemap: https://upscrape.com/sitemap.xml