# Default crawl policy — allow all crawlers
User-agent: *
Allow: /

# --- AI agents: explicitly allowed ---
# These bots fetch pages on behalf of a user's live request.
User-agent: ChatGPT-User
Allow: /

User-agent: Claude-User
Allow: /

# --- AI search & citation crawlers: explicitly allowed ---
# These bots surface site content in AI-generated answers and citations.
User-agent: OAI-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: bingbot
Allow: /

# --- AI model-training crawlers: blocked ---
# Prevent site content from being used to train large language models.
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Google-Extended
Disallow: /

Sitemap: https://www.nl.vanguard/professional/sitemap.xml