# robots.txt — art2link.com
# Last updated: 2026-05-27
# Strategy: open the site to AI training + AI-search crawlers (per Art2link AEO plan),
# while keeping admin, ticket portal, and EDI portal paths out of crawls.
#
# Note on robots.txt semantics: any User-agent with its own block IGNORES the wildcard (*)
# block. That's why disallows are restated inside each named-bot group below.

# ===== Default — applies to any crawler without its own block =====
User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /tickets/html/
Disallow: /edi-portal-ie/

# ===== Traditional search crawlers (render JS — need admin-ajax) =====
User-agent: Googlebot
User-agent: Bingbot
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /tickets/html/
Disallow: /edi-portal-ie/

# ===== AI training + AI-search crawlers =====
# OpenAI:    GPTBot (training)          · ChatGPT-User (user browsing)   · OAI-SearchBot (ChatGPT search index)
# Anthropic: ClaudeBot (training)       · Claude-User (user browsing)    · Claude-SearchBot (search index)
# Google:    Google-Extended (Gemini training — Googlebot above covers search)
# Perplexity:PerplexityBot (index)      · Perplexity-User (user fetches)
# Apple:     Applebot-Extended (Apple Intelligence training; Applebot also covers Siri/Spotlight)
# Meta:      Meta-ExternalAgent (Meta AI)
# Amazon:    Amazonbot (Alexa, Rufus)
# DuckDuckGo:DuckAssistBot
# Cohere:    cohere-ai
# Common Crawl: CCBot (corpus that most open-source LLMs train on — high leverage)
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: Google-Extended
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Meta-ExternalAgent
User-agent: Amazonbot
User-agent: DuckAssistBot
User-agent: cohere-ai
User-agent: CCBot
Allow: /
Disallow: /wp-admin/
Disallow: /tickets/html/
Disallow: /edi-portal-ie/

Sitemap: https://www.art2link.com/sitemap.xml