# robots.txt for Aegis CRA — https://aegiscra.com
# Policy: open to search and AI search; no AI model training.

User-agent: *
Allow: /
Disallow: /api/

# --- AI crawlers: allowed to read for search / grounding, not for training ---
User-agent: GPTBot
Disallow: /
# GPTBot is used for model training; blocked.

User-agent: OAI-SearchBot
Allow: /
# OpenAI search crawler; allowed.

User-agent: ChatGPT-User
Allow: /

User-agent: Google-Extended
Disallow: /
# Google-Extended governs Gemini training / grounding; blocked from training use.

User-agent: Applebot-Extended
Disallow: /

User-agent: ClaudeBot
Disallow: /
# ClaudeBot performs training crawls; blocked.

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Disallow: /

User-agent: CCBot
Disallow: /
# Common Crawl feeds many training sets; blocked.

User-agent: PerplexityBot
Allow: /

User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Allow: /

User-agent: cohere-ai
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

# --- Content Signals (contentsignals.org) ---
# Applies to content on this site regardless of the rules above.
Content-Signal: search=yes, ai-train=no, ai-input=no

Sitemap: https://aegiscra.com/sitemap.xml