# AARTHLABS - Motion Graphics & Video Production Studio
# https://aarthlabs.com

# ===========================================
# Standard Web Crawlers
# ===========================================

User-agent: *
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /_next/static/
Disallow: /_next/image/

# ===========================================
# AI Crawlers & Language Models
# ===========================================

# OpenAI GPTBot
User-agent: GPTBot
Allow: /
Allow: /about
Allow: /services
Allow: /case-studies
Allow: /testimonials
Allow: /faq
Allow: /pricing
Allow: /careers
Allow: /contact
Allow: /showcase
Allow: /privacy
Allow: /terms
Disallow: /api/
Disallow: /admin/

# Google Bard / Gemini
User-agent: Google-Extended
Allow: /
Allow: /about
Allow: /services
Allow: /case-studies
Allow: /testimonials
Allow: /faq
Allow: /pricing
Allow: /careers
Allow: /contact
Allow: /showcase
Disallow: /api/
Disallow: /admin/

# Anthropic Claude
User-agent: anthropic-ai
Allow: /
Allow: /about
Allow: /services
Allow: /case-studies
Allow: /testimonials
Allow: /faq
Allow: /pricing
Allow: /careers
Allow: /contact
Allow: /showcase
Disallow: /api/
Disallow: /admin/

User-agent: Claude-Web
Allow: /
Allow: /about
Allow: /services
Allow: /case-studies
Allow: /testimonials
Allow: /faq
Allow: /pricing
Allow: /careers
Allow: /contact
Allow: /showcase
Disallow: /api/
Disallow: /admin/

# Bing AI / Copilot
User-agent: Bingbot
Allow: /
Disallow: /api/
Disallow: /admin/

# Meta AI
User-agent: FacebookBot
Allow: /
Disallow: /api/
Disallow: /admin/

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Allow: /about
Allow: /services
Allow: /case-studies
Allow: /testimonials
Allow: /faq
Allow: /pricing
Allow: /careers
Allow: /contact
Allow: /showcase
Disallow: /api/
Disallow: /admin/

# Common Crawl (used for AI training)
User-agent: CCBot
Allow: /
Allow: /about
Allow: /services
Allow: /case-studies
Disallow: /api/
Disallow: /admin/

# Cohere AI
User-agent: cohere-ai
Allow: /
Allow: /about
Allow: /services
Allow: /case-studies
Allow: /testimonials
Disallow: /api/
Disallow: /admin/

# ===========================================
# Sitemap & AI.txt References
# ===========================================

Sitemap: https://aarthlabs.com/sitemap.xml
AI-txt: https://aarthlabs.com/ai.txt

# ===========================================
# Crawl Delay (be nice to our servers)
# ===========================================

Crawl-delay: 2