# =============================
# robots.txt — директивы для поисковых роботов
# RFC 9309 (Robot Exclusion Protocol). Sitemap: https://syntagma.app/sitemap.xml
# =============================

User-agent: *
Allow: /

# -----------------------------
# Исключаем технические разделы
# -----------------------------
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard/
Disallow: /static/private/
Disallow: /assets/private/
Disallow: /cgi-bin/
Disallow: /temp/
Disallow: /_next/
Disallow: /node_modules/
Disallow: /vendor/
Disallow: /oauth/callback
Disallow: /confirm-email
Disallow: /sign-in
Disallow: /sign-up
Disallow: /password-reset
Disallow: /password-reset/confirm

# Безопасность (общие для *)
Disallow: /config/
Disallow: /secrets/
Disallow: /env/
Disallow: /.git/
Disallow: /.env
Disallow: /.well-known/private/

# Внутренние описания для команды (*.scope.txt) — не индексировать
Disallow: /*.scope.txt
# Вложенные пути вроде /geo/*.scope.txt не покрываются одним сегментом в /*.scope.txt — явно закрываем каталоги только со scope
Disallow: /geo/
Disallow: /crawl/
Disallow: /verification/

# -----------------------------
# Хост и карта сайта
# -----------------------------
Host: syntagma.app
Sitemap: https://syntagma.app/sitemap.xml

# IndexNow (optional): verification key URL https://syntagma.app/e8f4c2d1a90b4e6f8a7c5d3e2f1098ab.txt — ping API after deploy when URLs change

# -----------------------------
# Основные поисковики (Crawl-delay не поддерживается Google/Bing)
# -----------------------------

# Googlebot
User-agent: Googlebot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /private/

# Яндекс (основной и дополнительные обходчики)
User-agent: Yandex
User-agent: YandexBot
User-agent: YandexAdditional
User-agent: YandexAdditionalBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Clean-param: ref&utm_source&utm_medium&utm_campaign&utm_term&utm_content&gclid&yclid /

# Bing / Microsoft
User-agent: bingbot
Allow: /

# DuckDuckGo
User-agent: DuckDuckBot
Allow: /

# Facebook / LinkedIn / Telegram link preview bots
User-agent: Facebot
User-agent: LinkedInBot
User-agent: TelegramBot
Allow: /

# -----------------------------
# LLM / AI обучение и выдача (явный доступ к публичному контенту)
# -----------------------------
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Google-Extended
User-agent: Googlebot-Extended
User-agent: PerplexityBot
User-agent: CCBot
User-agent: Amazonbot
User-agent: Applebot-Extended
User-agent: FacebookBot
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: YouBot
User-agent: NeevaBot
User-agent: AI2Bot
User-agent: Bytespider
User-agent: cohere-ai
User-agent: Diffbot
User-agent: omgili
User-agent: omgilibot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard/
Disallow: /static/private/
Disallow: /assets/private/
Disallow: /config/
Disallow: /secrets/
Disallow: /env/
Disallow: /.git/
Disallow: /.env
Disallow: /.well-known/private/
Disallow: /oauth/callback
Disallow: /confirm-email
Disallow: /sign-in
Disallow: /sign-up
Disallow: /password-reset
Disallow: /password-reset/confirm
Disallow: /*.scope.txt
Disallow: /geo/
Disallow: /crawl/
Disallow: /verification/