# Robots.txt for ZenCap - Premium Financial Models & Advisory
# Site: https://zencap.ai
# Last Updated: August 2025

# ================================
# LEGITIMATE SEARCH ENGINES - ALLOWED
# ================================

# NOTE: robots.txt groups do NOT inherit from each other. Any path that must be
# kept out of the index has to be repeated in EVERY User-agent group below,
# including the internal-tools list (kept in sync with INTERNAL_ADMIN_PAGES in
# src/middleware.ts).

# Google
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: AdsBot-Google
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /test-*
Disallow: /warrior
Disallow: /portfolios
Disallow: /monitoring
Disallow: /demo
Disallow: /codebase-graph
Disallow: /graph-data/
Disallow: /pc-build-prices
Disallow: /webhook-diagnostics
Disallow: /developers
Disallow: /meridian
Disallow: /meridian-search
Disallow: /analysis
Disallow: /analyst
# Fabricated-sample research pages — admin-gated 2026-08-05 (content compliance).
Disallow: /research/bull-bear
Disallow: /render
Crawl-delay: 1

# Bing
User-agent: Bingbot
User-agent: BingPreview
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /test-*
Disallow: /warrior
Disallow: /portfolios
Disallow: /monitoring
Disallow: /demo
Disallow: /codebase-graph
Disallow: /graph-data/
Disallow: /pc-build-prices
Disallow: /webhook-diagnostics
Disallow: /developers
Disallow: /meridian
Disallow: /meridian-search
Disallow: /analysis
Disallow: /analyst
# Fabricated-sample research pages — admin-gated 2026-08-05 (content compliance).
Disallow: /research/bull-bear
Disallow: /render
Crawl-delay: 1

# ================================
# SOCIAL MEDIA - ALLOWED (Critical for marketing)
# ================================

# Facebook/Meta (IMPORTANT - Keep this allowed!)
User-agent: facebookexternalhit
User-agent: facebookcatalog
User-agent: meta-externalagent
Allow: /
Disallow: /api/
Disallow: /admin/
# Fabricated-sample research pages — admin-gated 2026-08-05 (content compliance).
Disallow: /research/bull-bear

# LinkedIn (Important for B2B)
User-agent: LinkedInBot
Allow: /
Disallow: /api/
Disallow: /admin/
# Fabricated-sample research pages — admin-gated 2026-08-05 (content compliance).
Disallow: /research/bull-bear

# Twitter/X
User-agent: Twitterbot
Allow: /
Disallow: /api/
Disallow: /admin/
# Fabricated-sample research pages — admin-gated 2026-08-05 (content compliance).
Disallow: /research/bull-bear

# ================================
# AI BOTS - BLOCKED (Protect your $9,875-$14,750 models)
# ================================

# OpenAI
User-agent: GPTBot
User-agent: ChatGPT-User
Disallow: /

# Anthropic
User-agent: Claude-Web
User-agent: anthropic-ai
User-agent: ClaudeBot
Disallow: /

# Google AI
User-agent: Google-Extended
User-agent: GoogleOther
Disallow: /

# Common Crawl (used for AI training)
User-agent: CCBot
Disallow: /

# ByteDance/TikTok
User-agent: Bytespider
Disallow: /

# Perplexity AI
User-agent: PerplexityBot
Disallow: /

# ================================
# SEO TOOLS - BLOCKED (Protect competitive intel)
# ================================

User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: DotBot
User-agent: MJ12bot
User-agent: Majestic
User-agent: BLEXBot
User-agent: DataForSeoBot
Disallow: /

# ================================
# CONTENT SCRAPERS - BLOCKED
# ================================

User-agent: HTTrack
User-agent: WebCopier
User-agent: wget
User-agent: Scrapy
Disallow: /

# ================================
# DEFAULT RULES - ALL OTHER BOTS
# ================================

User-agent: *
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /test-*
Disallow: /debug-*
Disallow: /auth/
Disallow: /preview/
Disallow: /customer-portal
# Internal tools — auth-gated in src/middleware.ts (INTERNAL_ADMIN_PAGES).
# Listed here as defense in depth, and to de-index anything already crawled.
Disallow: /warrior
Disallow: /portfolios
Disallow: /monitoring
Disallow: /demo
Disallow: /codebase-graph
Disallow: /graph-data/
Disallow: /pc-build-prices
Disallow: /webhook-diagnostics
Disallow: /developers
Disallow: /meridian
Disallow: /meridian-search
Disallow: /analysis
Disallow: /analyst
# Fabricated-sample research pages — admin-gated 2026-08-05 (content compliance).
Disallow: /research/bull-bear
Disallow: /render
Disallow: /*.xlsx$
Disallow: /*.xlsm$
Disallow: /*.xls$
Disallow: /uploads/
Crawl-delay: 2

# Priority pages for legitimate crawlers
Allow: /models/
Allow: /research/
Allow: /solutions/
Allow: /about/
Allow: /contact/

# ================================
# SITEMAP
# ================================

Sitemap: https://zencap.ai/sitemap.xml

# ================================
# SPECIAL DIRECTIVES
# ================================

# Note: AI training prevention is handled by User-agent blocks above
# X-Robots-Tag directives must be served as HTTP headers, not in robots.txt