# FrontView Magazine — robots.txt
#
# NOTE: bot-blocking (403) for scrapers happens at the server WAF level,
# NOT here. Allowing a bot below only matters if the WAF also lets it through.

# ── Default: all crawlers ─────────────────────────────────────────────
User-agent: *
Disallow: /admin
Disallow: /login
Disallow: /logout
Disallow: /register
Disallow: /password
Disallow: /_profiler
Disallow: /_wdt

# Site search: an unbounded set of URLs, each one an Elasticsearch query.
Disallow: /nl/search
Disallow: /en/search

# Paginated listings. Every album, article and event has its own URL in the sitemap,
# so nothing needs to be discovered by walking pagination — and the access logs show
# crawlers requesting things like `?page=719` on an album with eight pages.
Disallow: /*?page=
Disallow: /*&page=

# ── AI retrieval / citation bots (these drive referral traffic) ───────
# Explicitly welcomed so they can fetch pages live and cite us with a link.
User-agent: OAI-SearchBot
Allow: /
Disallow: /admin

User-agent: ChatGPT-User
Allow: /
Disallow: /admin

User-agent: PerplexityBot
Allow: /
Disallow: /admin

User-agent: Perplexity-User
Allow: /
Disallow: /admin

User-agent: Claude-Web
Allow: /
Disallow: /admin

User-agent: Claude-User
Allow: /
Disallow: /admin

# ── AI training crawlers (no direct traffic; allowed for visibility) ──
User-agent: GPTBot
Allow: /
Disallow: /admin

User-agent: ClaudeBot
Allow: /
Disallow: /admin

User-agent: Google-Extended
Allow: /
Disallow: /admin

# ── SEO / backlink crawlers ───────────────────────────────────────────
# These send no readers and no citations, and they are the heaviest crawlers in the
# access logs. Advisory only — a bot that ignores robots.txt has to be stopped by the
# server WAF, which is where the 403s for these ranges already come from.
User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

Sitemap: https://frontview-magazine.be/sitemap.xml