# Default policy for all crawlers.
User-agent: *
Disallow: /admin
Disallow: /auth/login

# AI crawlers are explicitly allowed so this site can be surfaced and cited by
# ChatGPT, Claude, Perplexity and Gemini. This covers the retrieval/search bots
# (OAI-SearchBot, ChatGPT-User, Claude-SearchBot, Claude-User, PerplexityBot,
# Perplexity-User) and Google-Extended (Gemini grounding). GPTBot and ClaudeBot
# are general crawlers whose data may also feed model training — we allow them
# deliberately for AEO/brand presence. Only the pure-training corpora below
# (CCBot, Applebot-Extended) are opted out. Decision: VTM-452.
# Mirrors the default group's private-path disallows.
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
Disallow: /admin
Disallow: /auth/login

# Pure-training corpora with no citation benefit are opted out (VTM-452).
User-agent: CCBot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

Sitemap: https://www.valuethemarkets.com/sitemap.xml
Sitemap: https://www.valuethemarkets.com/sitemap-news.xml