# myssagent.com — robots.txt
# Sitemap is regenerated on every build by scripts/generate-sitemap.mjs.

# One group for every crawler we name, plus the wildcard. Listing the named crawlers in the
# same group as * means each of them gets the same Disallow rules. Before 2026-10-01 each
# named crawler had its own group containing only "Allow: /", and a crawler obeys only the
# most specific group that names it, so the private-surface Disallows below never applied to
# Googlebot, Bingbot, GPTBot or any of the others.
# Everything that was crawlable before is still crawlable; only the Disallow lines below changed scope.

# Search engines, social unfurlers, AI search crawlers (explicitly allowed for AEO visibility)
User-agent: Googlebot
User-agent: Bingbot
User-agent: Twitterbot
User-agent: facebookexternalhit
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: Claude-SearchBot
User-agent: anthropic-ai
User-agent: Google-Extended
User-agent: CCBot
User-agent: Applebot-Extended
User-agent: *
Allow: /

# Private surfaces
Disallow: /admin/
Disallow: /api/
Disallow: /demo-
Disallow: /demo
Disallow: /demo/
Disallow: /invest
Disallow: /share
Disallow: /aep

# Auth & account flows
Disallow: /auth
Disallow: /oauth/
Disallow: /reset-password
Disallow: /create-account
Disallow: /advisors/start
Disallow: /terms-agreement
Disallow: /dashboard

# Analyzer funnel (in-flow, flow-specific URLs)
Disallow: /start
Disallow: /marital-status
Disallow: /upload
Disallow: /work
Disallow: /unlock-report
Disallow: /processing
Disallow: /results
Disallow: /success

# Affiliate portal (auth-gated)
Disallow: /affiliate/login
Disallow: /affiliate/dashboard
Disallow: /affiliate/pending
Disallow: /affiliate/links
Disallow: /affiliate/deals
Disallow: /affiliate/activity
Disallow: /affiliate/payouts
Disallow: /affiliate/resources
Disallow: /affiliate/settings

# Internal / preview
Disallow: /testing
Disallow: /preview/strategies
Disallow: /report/preview/

Sitemap: https://myssagent.com/sitemap.xml