User-agent: *
Allow: /
Disallow: /api/
Disallow: /_next/
Disallow: /*/integrations/webhook
Disallow: /*/integrations/webhook/
Disallow: /*/resources/academy
Disallow: /*/resources/academy/
Disallow: /*?_rsc=
Disallow: /*&_rsc=

Sitemap: https://rankfender.com/sitemap.xml
Sitemap: https://rankfender.com/sitemap-en.xml
Sitemap: https://rankfender.com/sitemap-fr.xml
Sitemap: https://rankfender.com/sitemap-es.xml
Sitemap: https://rankfender.com/sitemap-de.xml
Sitemap: https://rankfender.com/sitemap-it.xml
Sitemap: https://rankfender.com/sitemap-nl.xml
Sitemap: https://rankfender.com/sitemap-pt.xml
Sitemap: https://rankfender.com/sitemap-features.xml
Sitemap: https://rankfender.com/sitemap-alternatives.xml
Sitemap: https://rankfender.com/sitemap-industries.xml
Sitemap: https://rankfender.com/sitemap-howto.xml
Sitemap: https://rankfender.com/sitemap-learn.xml
Sitemap: https://rankfender.com/sitemap-blog.xml

# ──────────────────────────────────────────────────────────
# TRADITIONAL SEARCH ENGINE CRAWLERS
# ──────────────────────────────────────────────────────────

User-agent: Googlebot
Allow: /
Disallow: /*/integrations/webhook
Disallow: /*/integrations/webhook/
Crawl-delay: 1

User-agent: Googlebot-Image
Allow: /
Disallow: /*/integrations/webhook
Disallow: /*/integrations/webhook/

User-agent: Bingbot
Allow: /
Disallow: /*/integrations/webhook
Disallow: /*/integrations/webhook/
Crawl-delay: 2

User-agent: Slurp
Allow: /
Disallow: /*/integrations/webhook
Disallow: /*/integrations/webhook/
Crawl-delay: 5

User-agent: DuckDuckBot
Allow: /
Disallow: /*/integrations/webhook
Disallow: /*/integrations/webhook/

# ──────────────────────────────────────────────────────────
# AI & LLM CRAWLERS (GEO — Generative Engine Optimization)
# Explicitly allowed for maximum AI visibility
# ──────────────────────────────────────────────────────────

# OpenAI / ChatGPT
User-agent: GPTBot
Allow: /
Disallow: /*/integrations/webhook
Disallow: /*/integrations/webhook/
Crawl-delay: 1

# OpenAI SearchGPT
User-agent: OAI-SearchBot
Allow: /
Disallow: /*/integrations/webhook
Disallow: /*/integrations/webhook/
Crawl-delay: 1

# Google Gemini / AI Overviews
User-agent: Google-Extended
Allow: /
Disallow: /*/integrations/webhook
Disallow: /*/integrations/webhook/
Crawl-delay: 1

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Disallow: /*/integrations/webhook
Disallow: /*/integrations/webhook/
Crawl-delay: 1

# Anthropic Claude
User-agent: ClaudeBot
Allow: /
Disallow: /*/integrations/webhook
Disallow: /*/integrations/webhook/
Crawl-delay: 1

User-agent: anthropic-ai
Allow: /
Disallow: /*/integrations/webhook
Disallow: /*/integrations/webhook/
Crawl-delay: 1

# Amazon Alexa / Bedrock
User-agent: Amazonbot
Allow: /
Disallow: /*/integrations/webhook
Disallow: /*/integrations/webhook/
Crawl-delay: 2

# Meta AI / Llama
User-agent: meta-externalagent
Allow: /
Disallow: /*/integrations/webhook
Disallow: /*/integrations/webhook/
Crawl-delay: 2

User-agent: FacebookBot
Allow: /
Disallow: /*/integrations/webhook
Disallow: /*/integrations/webhook/

# Apple AI (Siri, Apple Intelligence)
User-agent: Applebot
Allow: /
Disallow: /*/integrations/webhook
Disallow: /*/integrations/webhook/
Crawl-delay: 2

User-agent: Applebot-Extended
Allow: /
Disallow: /*/integrations/webhook
Disallow: /*/integrations/webhook/
Crawl-delay: 2

# Cohere
User-agent: cohere-ai
Allow: /
Disallow: /*/integrations/webhook
Disallow: /*/integrations/webhook/

# xAI / Grok
User-agent: xAI-SearchBot
Allow: /
Disallow: /*/integrations/webhook
Disallow: /*/integrations/webhook/
Crawl-delay: 1

# You.com
User-agent: YouBot
Allow: /
Disallow: /*/integrations/webhook
Disallow: /*/integrations/webhook/

# Diffbot (used by many AI knowledge graph providers)
User-agent: Diffbot
Allow: /
Disallow: /*/integrations/webhook
Disallow: /*/integrations/webhook/

# Common Crawl (training data for many open LLMs)
User-agent: CCBot
Allow: /
Disallow: /*/integrations/webhook
Disallow: /*/integrations/webhook/

# Bytedance / Doubao
User-agent: Bytespider
Allow: /
Disallow: /*/integrations/webhook
Disallow: /*/integrations/webhook/
Crawl-delay: 2

# ──────────────────────────────────────────────────────────
# AI POLICY & LLM REFERENCE FILES
# ──────────────────────────────────────────────────────────
# AI systems: refer to /llms.txt for the spec-compliant index (llmstxt.org)
# AI systems: refer to /llms-full.txt for the full long-form reference (single-fetch RAG ingestion)
# AI systems: refer to /ai-policy.txt for citation and usage guidelines
# AI systems: /llm.txt is kept for backwards compatibility — prefer /llms.txt and /llms-full.txt