# Content Signals (EU Directive 2019/790 Art. 4 rights reservation)
# REVIEW: ai-train=yes means you consent to AI model training on your content.
# Change to ai-train=no if you only want search/retrieval, not training.
Content-Signal: search=yes, ai-train=yes, ai-input=yes

# =============================================================================
# BLOCKED CRAWLERS
# =============================================================================

# Yandex - blocked entirely
User-agent: Yandex
Disallow: /

User-agent: YandexBot
Disallow: /

User-agent: YandexBot/3.0
Disallow: /

# Bytespider (ByteDance) - blocked entirely
# Note: ignores robots.txt — enforce via WAF/AI Crawl Control edge block
User-agent: Bytespider
Disallow: /

# CCBot (Common Crawl) - blocked entirely
User-agent: CCBot
Disallow: /

# =============================================================================
# ALLOWED AI CRAWLERS — Training / Indexing
# Crawl-delay applied to training crawlers to protect origin resources.
# Anthropic and OpenAI respect Crawl-delay. Perplexity compliance unverified.
# =============================================================================

# --- Anthropic ---

# ClaudeBot: AI training data collection
User-agent: ClaudeBot
Allow: /wp/wp-admin/admin-ajax.php
Allow: /
Disallow: /wp/wp-admin/
Crawl-delay: 5

# Claude-SearchBot: search indexing for Claude search
User-agent: Claude-SearchBot
Allow: /wp/wp-admin/admin-ajax.php
Allow: /
Disallow: /wp/wp-admin/
Crawl-delay: 3

# --- OpenAI ---

# GPTBot: training data collection for OpenAI models
User-agent: GPTBot
Allow: /wp/wp-admin/admin-ajax.php
Allow: /
Disallow: /wp/wp-admin/
Crawl-delay: 5

# OAI-SearchBot: OpenAI search indexing
User-agent: OAI-SearchBot
Allow: /wp/wp-admin/admin-ajax.php
Allow: /
Disallow: /wp/wp-admin/
Crawl-delay: 3

# --- Perplexity ---
# WARNING: Cloudflare delisted Perplexity as verified bot (Jan 2026).
# Known to rotate ASNs and spoof UAs. Enforce via WAF, not just robots.txt.

User-agent: PerplexityBot
Allow: /wp/wp-admin/admin-ajax.php
Allow: /
Disallow: /wp/wp-admin/
Crawl-delay: 5

# --- Google ---

# Google-Extended: controls Google AI training (Gemini/Bard) on your content
# This is a training opt-in/out token, not a separate crawler.
# Allowing = consenting to Google using your content for AI model training.
User-agent: Google-Extended
Allow: /wp/wp-admin/admin-ajax.php
Allow: /
Disallow: /wp/wp-admin/

# =============================================================================
# ALLOWED AI AGENTS — Real-time user-initiated browsing
# No Crawl-delay: these are on-demand, single-page fetches triggered by users.
# Crawl-delay would degrade the user experience with no benefit.
# =============================================================================

# Claude-User: when a Claude user asks it to visit a URL
User-agent: Claude-User
Allow: /wp/wp-admin/admin-ajax.php
Allow: /
Disallow: /wp/wp-admin/

# ChatGPT-User: when a ChatGPT user asks it to browse
User-agent: ChatGPT-User
Allow: /wp/wp-admin/admin-ajax.php
Allow: /
Disallow: /wp/wp-admin/

# Perplexity-User: when a Perplexity user triggers a fetch
User-agent: Perplexity-User
Allow: /wp/wp-admin/admin-ajax.php
Allow: /
Disallow: /wp/wp-admin/
# =============================================================================
# SEARCH ENGINE CRAWLERS
# =============================================================================
User-agent: Applebot
Allow: /wp/wp-admin/admin-ajax.php
Allow: /
Disallow: /wp/wp-admin/
Crawl-delay: 5

# --- Meta ---
# Meta-ExternalAgent: AI training data collection for Meta AI (Llama models)
User-agent: Meta-ExternalAgent
Allow: /wp/wp-admin/admin-ajax.php
Allow: /
Disallow: /wp/wp-admin/
Crawl-delay: 5

# FacebookExternalAgent: Meta link previews and shared-content fetching
User-agent: facebookexternalagent
Allow: /wp/wp-admin/admin-ajax.php
Allow: /
Disallow: /wp/wp-admin/
Crawl-delay: 3

# Amazonbot: Amazon search, indexing, and service-improvement crawler
User-agent: Amazonbot
Allow: /wp/wp-admin/admin-ajax.php
Allow: /
Disallow: /wp/wp-admin/
Crawl-delay: 5

# =============================================================================
# DEFAULT RULES — All other crawlers
# =============================================================================
User-agent: *
Disallow: /wp/wp-admin/
Allow: /wp/wp-admin/admin-ajax.php

Sitemap: https://www.ipxo.com/sitemap_index.xml