# robots.txt for www.slavin.ai
# Last updated: 2026-05-11

# === Search engines ===
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Yandex
Allow: /

User-agent: DuckDuckBot
Allow: /

# === AI training & inference bots (explicit allow) ===
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: Bytespider
Allow: /

User-agent: DiffBot
Allow: /

# === Catch-all ===
User-agent: *
Disallow: /Admin/
Disallow: /API/
Disallow: /api/
Disallow: /bin/
Disallow: /obj/
Disallow: /App_Data/
Disallow: /tmp/
Disallow: /Test.aspx
Disallow: /test.aspx
Allow: /

# === Sitemap ===
Sitemap: https://www.slavin.ai/sitemap.xml
# Atom feed of the most recently changed pages. Google and Bing accept
# RSS/Atom as a sitemap format, and a feed is the freshness half of what
# sitemap.xml says once: it has been generated at application start since
# 2026-06 and nothing pointed at it until 2026-10-14.
Sitemap: https://www.slavin.ai/feed.xml