# Claw Ecosystem - Robots.txt
# Optimized for AI agents and search engines

# Allow all standard crawlers
User-agent: *
Allow: /

# Specific agent rules
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 1

# AI Agent crawlers - encouraged
User-agent: Perplexity-bot
Allow: /api/
Allow: /agent-access/
Crawl-delay: 0.5

User-agent: *bot
Allow: /api/
Allow: /agent-access/
Disallow: /dashboard/
Crawl-delay: 1

# Sitemap for discovery
Sitemap: https://claw-news.com/sitemap.xml

# Agent manifest location
Agent-Access: /agent-access/manifest.json

# API documentation
API-Docs: /agents/api-documentation.md

# Rate limit info
X-RateLimit: 100 requests/minute