# ==============================================================================
# robots.txt for SHPForge Platform (https://shpforge.wicak.id)
# Optimized for Google, Bing, Yahoo, and Modern AI Crawlers (OpenAI, Anthropic, Perplexity)
# ==============================================================================

# 1. Global Search Crawlers
User-agent: *
Allow: /
Allow: /features
Allow: /architecture
Allow: /pricing
Allow: /terms
Allow: /privacy
Allow: /sitemap.xml
Allow: /robots.txt
Allow: /favicon.ico
Allow: /favicon.svg
Allow: /favicon-*.png
Allow: /apple-touch-icon.png
Allow: /android-chrome-*.png
Allow: /og-image.png
Allow: /site.webmanifest
Allow: /css/
Allow: /js/

# Disallow Private Console, User Sessions, Internal APIs & Chat
Disallow: /admin/
Disallow: /dashboard
Disallow: /profile
Disallow: /devices/
Disallow: /tools
Disallow: /utilities
Disallow: /chat
Disallow: /billing
Disallow: /api/
Disallow: /login
Disallow: /register
Disallow: /verify-email/
Disallow: /logout

# 2. Googlebot (Google Search & Discover)
User-agent: Googlebot
Allow: /
Allow: /features
Allow: /architecture
Allow: /pricing
Allow: /terms
Allow: /privacy
Allow: /sitemap.xml
Disallow: /admin/
Disallow: /dashboard
Disallow: /profile
Disallow: /devices/
Disallow: /tools
Disallow: /utilities
Disallow: /chat
Disallow: /api/
Disallow: /login
Disallow: /register

# 3. Google Gemini / Vertex AI Training & Knowledge
User-agent: Google-Extended
Allow: /
Allow: /features
Allow: /architecture
Allow: /pricing
Allow: /terms
Allow: /privacy
Disallow: /admin/
Disallow: /dashboard
Disallow: /profile
Disallow: /chat
Disallow: /api/

# 4. OpenAI / ChatGPT Search & Browsing (GPTBot & ChatGPT-User)
User-agent: GPTBot
Allow: /
Allow: /features
Allow: /architecture
Allow: /pricing
Allow: /terms
Allow: /privacy
Disallow: /admin/
Disallow: /dashboard
Disallow: /profile
Disallow: /chat
Disallow: /api/

User-agent: ChatGPT-User
Allow: /
Allow: /features
Allow: /architecture
Allow: /pricing
Allow: /terms
Allow: /privacy
Disallow: /admin/
Disallow: /dashboard
Disallow: /profile
Disallow: /chat
Disallow: /api/

# 5. Anthropic Claude (ClaudeBot & anthropic-ai)
User-agent: ClaudeBot
Allow: /
Allow: /features
Allow: /architecture
Allow: /pricing
Allow: /terms
Allow: /privacy
Disallow: /admin/
Disallow: /dashboard
Disallow: /profile
Disallow: /chat
Disallow: /api/

User-agent: anthropic-ai
Allow: /
Allow: /features
Allow: /architecture
Allow: /pricing
Allow: /terms
Allow: /privacy
Disallow: /admin/
Disallow: /dashboard
Disallow: /profile
Disallow: /chat
Disallow: /api/

# 6. Perplexity AI Search Engine
User-agent: PerplexityBot
Allow: /
Allow: /features
Allow: /architecture
Allow: /pricing
Allow: /terms
Allow: /privacy
Disallow: /admin/
Disallow: /dashboard
Disallow: /profile
Disallow: /chat
Disallow: /api/

# 7. Applebot (Apple Intelligence & Siri Web Search)
User-agent: Applebot
Allow: /
Allow: /features
Allow: /architecture
Allow: /pricing
Allow: /terms
Allow: /privacy
Disallow: /admin/
Disallow: /dashboard
Disallow: /profile
Disallow: /chat
Disallow: /api/

# 8. Sitemap Reference
Sitemap: https://shpforge.wicak.id/sitemap.xml