User-agent: *
Allow: /
Allow: /privacy
Allow: /terms
Allow: /about
Allow: /compare
Allow: /faq
Allow: /colleges-for-low-gpa
Allow: /colleges-for-high-sat
Allow: /best-colleges-for-engineering
Allow: /best-colleges-for-business
Allow: /best-colleges-for-nursing
Allow: /test-optional-colleges
Allow: /affordable-colleges
Allow: /universities
Allow: /universities/
Allow: /content
Allow: /glossary/acceptance-rate
Allow: /glossary/reach-target-safety
Allow: /glossary/ipeds
Allow: /glossary/early-decision-vs-early-action
Allow: /glossary/net-price
Allow: /guides/college-application-timeline
Allow: /guides/how-to-write-college-essay
Allow: /guides/financial-aid-letter
Allow: /guides/how-many-colleges-to-apply
Allow: /sitemap.xml
Allow: /sitemap-pages.xml
Allow: /sitemap-universities.xml
Allow: /llms.txt
Disallow: /api/

# AI Search Crawlers (allowed) — drive referral traffic
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: YouBot
Allow: /

User-agent: DuckAssistBot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Aggressive Scrapers (blocked) — training scrapers with no traffic benefit
User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Diffbot
Disallow: /

Sitemap: https://admit-coach.com/sitemap.xml