# ===============================================================
# General Rules for All Crawlers
# ===============================================================
User-agent: *

# Block backend and administrative areas
Disallow: /admin/

# Block user-specific, non-public pages
Disallow: /user/

# Block internal search results to avoid duplicate content
Disallow: /search/
Disallow: /*?s=
Disallow: /*&q=

# Block Next.js internal data fetching URLs and API routes
Disallow: /_next/data/
Disallow: /api/

# ===============================================================
# Rules for AI and Large Language Models
# ===============================================================

# Training crawlers — allow (open data philosophy)
User-agent: Google-Extended
Disallow:

User-agent: GPTBot
Disallow:

# Retrieval / RAG crawlers — explicitly allow for GEO visibility
User-agent: PerplexityBot
Disallow:

User-agent: Perplexity-User
Disallow:

User-agent: OAI-SearchBot
Disallow:

User-agent: ChatGPT-User
Disallow:

User-agent: Claude-User
Disallow:

User-agent: Applebot
Disallow:

User-agent: Amazonbot
Disallow:

# ===============================================================
# Optional: Block Specific Crawlers
# ===============================================================

# Example: To block Common Crawl, uncomment the following two lines.
# User-agent: CCBot
# Disallow: /


# ===============================================================
# Sitemap Location
# ===============================================================
Sitemap: https://tractordata.site/sitemap.xml