# https://www.robotstxt.org/robotstxt.html
# Optimized for Search Engines and AI Tools
#
# NOTE robots.txt semantics: a crawler uses ONLY its most specific matching
# User-agent group. Googlebot/Bingbot deliberately have NO own group here so
# they follow the `*` group below INCLUDING its Disallows (an own `Allow: /`
# group would make them ignore every Disallow). Social scrapers and AI answer
# engines keep explicit full-access groups (OG cards / citations); AI TRAINING
# crawlers get their own Disallow groups.
# Social preview bots — need full access for link previews
User-agent: Twitterbot
Allow: /
User-agent: facebookexternalhit
Allow: /
# AI TRAINING crawlers — fully disallowed (2026-08-10).
# They bring neither search visibility nor referral traffic, and they competed
# with Googlebot for a render pipeline that was already 18x oversubscribed.
# nginx additionally answers them 403 ($ai_blocked_ua) — robots.txt stays
# reachable for exactly that reason: a compliant crawler reads this and stops,
# while a bot that only ever sees 403 keeps retrying.
# KEEP THIS LIST AND $ai_blocked_ua IN nginx.conf IN SYNC.
User-agent: GPTBot
Disallow: /
User-agent: CCBot
Disallow: /
User-agent: anthropic-ai
Disallow: /
User-agent: Amazonbot
Disallow: /
User-agent: Bytespider
Disallow: /
# AI answer engines — may crawl, but nginx serves them the static shell instead
# of a Chrome render ($is_bot = 0). They are allowed here on purpose: a Disallow
# would also remove the site from their citations, and crawling costs us almost
# nothing now that no render is involved.
User-agent: ClaudeBot
Allow: /
User-agent: PerplexityBot
Allow: /
User-agent: OAI-SearchBot
Allow: /
# ChatGPT-User is a human pasting a link into ChatGPT, not a crawler — it keeps
# both full access and a full prerender.
User-agent: ChatGPT-User
Allow: /
# Robots.txt TOKENS, not crawlers: these two never appear as a User-Agent, they
# only opt content in/out of Google's and Apple's AI training. Blocking them in
# nginx would be decoration; the choice belongs here.
# Google-Extended has NO effect on Google Search indexing.
User-agent: Google-Extended
Disallow: /
User-agent: Applebot-Extended
Disallow: /
# Search engines + all other bots.
# The Disallow list mirrors the nginx $no_prerender denylist (app/auth-only
# routes that render as an empty JS shell for bots — crawling them wastes
# budget and used to create indexable empty pages). Keep the two lists in sync.
# Deliberately NOT disallowed: /feed (prerenders with noindex — bots must be
# able to SEE the noindex), /discover-creators, /post/, /ib/ (public content).
User-agent: *
Allow: /
Disallow: /api/
Disallow: /admin/
# /_next/ (old Next.js image proxy, gone since the 06/2026 redesign) is
# deliberately NOT disallowed anymore: nginx answers 410 so Google can SEE
# the removal and drop the stale image entries instead of retrying forever.
Disallow: /private/
# Landing pages (Paid-Traffic-Prelander): reine Conversion-Seiten, noindex
Disallow: /lp/
Disallow: /auth
Disallow: /verify-email
Disallow: /reset-password
Disallow: /confirm-email
Disallow: /delete-account
Disallow: /choose-creator-name
Disallow: /unsubscribe
Disallow: /profile
Disallow: /saved
Disallow: /tokens
# Kaufseite liegt seit 08/2026 hinter Login (RequireAuthRoute) — Bots sehen
# nur noch den Redirect auf /auth. Aus marketing-sitemap.xml entfernt.
Disallow: /credits
Disallow: /playground
Disallow: /create-images
Disallow: /text-to-audio
Disallow: /image-to-video
Disallow: /creator-studio
Disallow: /chat
Disallow: /settings
Disallow: /affiliate-dashboard
Disallow: /apply-status
Disallow: /creator-dashboard
Disallow: /redeem
Disallow: /claim
Disallow: /invite
Disallow: /backstage
Disallow: /advertiser
Disallow: /go/
Disallow: /interstitial/
# SEO Enhancement - Sitemaps
Sitemap: https://xipics.com/sitemap.xml
Sitemap: https://xipics.com/marketing-sitemap.xml
Sitemap: https://xipics.com/blog-sitemap.xml
Sitemap: https://xipics.com/content-sitemap
Sitemap: https://xipics.com/content-sitemap-nsfw
Sitemap: https://xipics.com/tag-sitemap
Sitemap: https://xipics.com/interactive-sitemap
Sitemap: https://xipics.com/story-sitemap
Sitemap: https://xipics.com/character-sitemap
Sitemap: https://xipics.com/creator-sitemap
# Crawl-delay for polite crawling (ignored by Google; keeps other bots from
# bursting the prerender). Bing honors it — 1 req/s is still ~86k URLs/day.
Crawl-delay: 1