# OperatorMesh robots.txt
# Block all bots from API endpoints — these are serverless functions
# that cost money and shouldn't be crawled

User-agent: *
Disallow: /.netlify/
Disallow: /api/
Disallow: /analyze
Disallow: /webhook
Disallow: /dashboard.html
Disallow: /login.html

# Allow all public pages
Allow: /
Allow: /index.html
Allow: /pricing.html
Allow: /integrations.html
Allow: /benchmark.html
Allow: /security.html
Allow: /feedback.html
Allow: /changelog.html
Allow: /roadmap.html
Allow: /about.html
Allow: /docs.html
Allow: /how-it-works.html
Allow: /customers.html
Allow: /privacy.html
Allow: /terms.html
Allow: /roi.html

# Block AI training bots explicitly
User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: FacebookBot
Disallow: /

# Block aggressive SEO crawlers (excluding Ahrefs — used for our own site audits)
User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

# Sitemap
Sitemap: https://operatormesh.com/sitemap.xml