User-agent: *
Crawl-delay: 5
Content-Signal: search=yes, ai-train=yes, ai-retrieval=yes
Disallow: /*/search/
Disallow: /*/login/?*
Disallow: /*/signup/?*

# AI crawlers — allow full access
User-agent: ChatGPT-User
Allow: /
Disallow: /*/search/
Disallow: /*/login/?*
Disallow: /*/signup/?*

User-agent: GPTBot
Allow: /
Disallow: /*/search/
Disallow: /*/login/?*
Disallow: /*/signup/?*

# ChatGPT live search index (distinct from GPTBot)
User-agent: OAI-SearchBot
Allow: /
Disallow: /*/search/
Disallow: /*/login/?*
Disallow: /*/signup/?*

User-agent: PerplexityBot
Allow: /
Disallow: /*/search/
Disallow: /*/login/?*
Disallow: /*/signup/?*

# Perplexity user-initiated fetch (companion to PerplexityBot)
User-agent: Perplexity-User
Allow: /
Disallow: /*/search/
Disallow: /*/login/?*
Disallow: /*/signup/?*

User-agent: ClaudeBot
Allow: /
Disallow: /*/search/
Disallow: /*/login/?*
Disallow: /*/signup/?*

# Gemini / AI Overviews grounding (does NOT affect Google Search rank)
User-agent: Google-Extended
Allow: /

User-agent: Google-CloudVertexBot
Allow: /

# Apple Intelligence, Amazon AI, Meta AI
User-agent: Applebot-Extended
Allow: /

User-agent: Amazonbot
Allow: /
Disallow: /*/search/

User-agent: Meta-ExternalAgent
Allow: /
Disallow: /*/search/
Disallow: /*/login/?*
Disallow: /*/signup/?*
# Its renderer requests "<page>/null" (32k/week, no such route)
Disallow: /*/null$
Disallow: /*/null/$

# Un-throttle Bing (feeds ChatGPT + Copilot); Google ignores crawl-delay
User-agent: Bingbot
Allow: /
Crawl-delay: 1

# Common Crawl (feeds many AI training sets) — full opt-in
User-agent: CCBot
Allow: /
Disallow: /*/search/
Disallow: /*/login/?*
Disallow: /*/signup/?*

# Block aggressive / low-Western-value crawler
User-agent: Bytespider
Disallow: /

Sitemap: https://remont.md/sitemap.xml