# AnalysisGen by VibeMatrix AI - robots.txt
# Optimized for AI crawlers and search engines
# Last updated: 2025-11-29

# Allow all AI crawlers and search engines
User-agent: GPTBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Googlebot
Allow: /
Crawl-delay: 0

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: YandexBot
Allow: /

User-agent: Applebot
Allow: /

User-agent: Sogou
Allow: /

User-agent: *
Allow: /

# Allow crawling of next.js static assets (CSS, JS, images)
Allow: /_next/static/

# Disallow crawling of internal API routes (except sitemap)
Disallow: /api/
Allow: /api/sitemap

# Disallow crawling of next.js build internals
Disallow: /.next/

# Disallow crawling of private pages
Disallow: /payment-success

# Disallow invented OpenSearch / SearchAction template litter (historical)
Disallow: /*?q=*
Disallow: /*?*search_term_string*

# Crawl delay for all bots
Crawl-delay: 0

# Request rate (requests per second)
Request-rate: 10/1s

# Sitemaps
Sitemap: https://www.vibematrixai.com/sitemap.xml
Sitemap: https://www.vibematrixai.com/api/sitemap