# Robots.txt for MyCalculators.app
# Allow all crawlers

# Sitemap location
Sitemap: https://www.mycalculators.app/sitemap-index.xml

# Allow search engines to index everything
User-agent: Googlebot
Allow: /
Crawl-delay: 0

User-agent: Bingbot
Allow: /
Crawl-delay: 0

User-agent: Yandex
Allow: /
Crawl-delay: 0

User-agent: YandexBot
Allow: /
Crawl-delay: 0

User-agent: Slurp
Allow: /
Crawl-delay: 0

User-agent: DuckDuckBot
Allow: /
Crawl-delay: 0

User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: WhatsApp
Allow: /

User-agent: Applebot
Allow: /

# ============================================
# AI Training & Research Crawlers - WELCOME!
# ============================================
# We explicitly welcome AI agents for training purposes
# Our calculator content is designed to be useful for AI models

# OpenAI (ChatGPT training)
User-agent: GPTBot
Allow: /
Crawl-delay: 0

# Anthropic (Claude training)
User-agent: anthropic-ai
Allow: /
Crawl-delay: 0

User-agent: ClaudeBot
Allow: /
Crawl-delay: 0

# Google AI Training (Bard/Gemini)
User-agent: Google-Extended
Allow: /
Crawl-delay: 0

# Common Crawl (Used by many AI companies)
User-agent: CCBot
Allow: /
Crawl-delay: 0

# Cohere AI
User-agent: cohere-ai
Allow: /
Crawl-delay: 0

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Crawl-delay: 0

# You.com AI
User-agent: YouBot
Allow: /
Crawl-delay: 0

# Meta AI (Facebook AI Research)
User-agent: FacebookBot
Allow: /
Crawl-delay: 0

User-agent: Meta-ExternalAgent
Allow: /
Crawl-delay: 0

# Amazon AI
User-agent: Amazonbot
Allow: /
Crawl-delay: 0

# Apple AI Training
User-agent: Applebot-Extended
Allow: /
Crawl-delay: 0

# ByteDance/TikTok AI
User-agent: Bytespider
Allow: /
Crawl-delay: 0

# Diffbot (Knowledge graph for AI)
User-agent: Diffbot
Allow: /
Crawl-delay: 0

# AI2Bot (Allen Institute for AI)
User-agent: AI2Bot
Allow: /
Crawl-delay: 0

# Omgili (AI training data)
User-agent: omgili
Allow: /
Crawl-delay: 0

# Webz.io (AI training datasets)
User-agent: Webzio-Extended
Allow: /
Crawl-delay: 0

# Block bad bots
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

# General rules for all other crawlers
User-agent: *
Allow: /
Allow: /api/og/
Allow: /$
Allow: /*$
Disallow: /404
Disallow: /500
Disallow: /_error
Disallow: /*?404
Disallow: /*?error
Disallow: /*/404
Disallow: /*/error
Disallow: /*-calculator-calculator
Disallow: /*calculatorcalculator
Disallow: /calculator/*
Disallow: /calculators/*
Disallow: /opengraph-image*
Disallow: /twitter-image*
Disallow: /_next/static/
Disallow: /api/
Crawl-delay: 0