# robots.txt for grin.com
# ----------------------------------------
# KI- und LLM-Bots (Lesen erlaubt, Training untersagt)
# ----------------------------------------
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-Web
User-agent: Gemini-Deep-Research
User-agent: Bard
User-agent: PerplexityBot
User-agent: PerplexityAI
User-agent: MistralBot
User-agent: CohereBot
User-agent: CCBot
User-agent: YouBot
User-agent: OpenDevin
User-agent: FirecrawlAgent
User-agent: DeepSeek
Allow: /document/
Disallow: /search/
AI-Train: no
AI-Input: yes
AI-Search: yes

# ----------------------------------------
# Nicht gewünschte oder irrelevante Bots
# ----------------------------------------
User-agent: PetalBot
Disallow: /

# ----------------------------------------
# Google Bots (volle Crawling-Erlaubnis)
# ----------------------------------------
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Mediapartners-Google
Allow: /

User-agent: Feedfetcher
Allow: /

# ----------------------------------------
# Weitere Suchmaschinen & SEO-Tools
# ----------------------------------------
User-agent: bingbot
Disallow: /search/
Disallow: /search?*

User-agent: AhrefsBot
Disallow: /search/
Disallow: /search?*

User-agent: SemrushBot
Disallow: /search/
Disallow: /search?*

User-agent: DotBot
Disallow: /search/
Disallow: /search?*

User-agent: YandexBot
Disallow: /search/
Disallow: /search?*

User-agent: Baiduspider
Disallow: /search/
Disallow: /search?*

User-agent: Bytespider
Disallow: /search/
Disallow: /search?*

# ----------------------------------------
# Globale Regeln (alle anderen Bots)
# ----------------------------------------
User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Allow: /document/
Disallow: /search/
Disallow: /search?*

# Informelles Signal für KI-Systeme
Content-Signal: /document/ ai-train=no, search=yes, ai-input=yes

# ----------------------------------------
# Sitemap
# ----------------------------------------
Sitemap: https://www.grin.com/xmlsitemap/index.xml