# RBX Invest — Roblox inventory and reseller analytics platform
# https://rbxinvest.com

User-agent: *
Allow: /
# Content Signals Policy (contentsignals.org), the vocabulary Cloudflare's AI
# Crawl Control reads. Permissive on purpose: every figure here is aggregated
# public Roblox data, being cited by an assistant is the distribution this
# site wants, and there is no paywalled corpus to protect.
#   search   = may be used to build a classic search index
#   ai-input = may be retrieved at answer time and cited (RAG)
#   ai-train = may be used as model training data
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# Block private/authenticated routes from indexing.
#
# /resell and /discord are NOT here. Both are linked from the site nav, and
# a Disallow on a linked page just gets it reported as "blocked from crawling"
# (SEMrush, Aug 2026) while leaving Google free to index the URL from anchor
# text alone. /resell is a deliberate public landing page: src/pages/Resell.tsx
# renders ResellPaywall with its own keywords and pricing copy for signed-out
# visitors, so blocking it was throwing away the page it was built to be.
# /discord is account plumbing and carries a noindex meta instead, which is the
# tag that actually deindexes; a Disallow would stop crawlers ever reading it.
# The Auth.md discovery document (/auth.md) sits under the /auth prefix the
# rules below block, and robots.txt matching is longest-match, so it needs
# an explicit Allow in every group carrying that Disallow. It is the file an
# agent reads to learn that public data needs no credentials; a crawler that
# obeyed the prefix block would never see it. It carries X-Robots-Tag: noindex
# from public/_headers, so allowing the fetch does not put it in an index.
Allow: /auth.md
Disallow: /admin
Disallow: /admin/
Disallow: /profile
Disallow: /purchases
Disallow: /auth
Disallow: /roblox-callback

# Major search bots
User-agent: Googlebot
Allow: /
Allow: /auth.md
Disallow: /admin
Disallow: /admin/
Disallow: /profile
Disallow: /purchases
Disallow: /auth
Disallow: /roblox-callback

User-agent: Bingbot
Allow: /
Allow: /auth.md
Disallow: /admin
Disallow: /admin/
Disallow: /profile
Disallow: /purchases
Disallow: /auth
Disallow: /roblox-callback

# Social link preview bots
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

# AI search bots. Allow citations and summaries of public pages.
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: GoogleOther
Allow: /

User-agent: CCBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: DuckAssistBot
Allow: /

User-agent: MistralAI-User
Allow: /

# LLM-oriented site index (llmstxt.org): curated Markdown map of key pages,
# articles, and top items. Regenerated on every build by scripts/prerender.mjs.
# https://rbxinvest.com/llms.txt
#
# Markdown for Agents: every /item/:assetId, /creator/:creatorId and
# /player/:userId page, and the pages llms.txt lists under "Key pages", are
# available as Markdown —
# append `.md` to the path, or send `Accept: text/markdown` to the normal URL.
# Responses carry an x-markdown-tokens estimate and open with a YAML
# front-matter block of machine-readable figures. Those URLs are noindex: they
# are for agents, not for search. See src/lib/markdownEntity.mjs.

# The dynamic child sitemaps live on api.rbxinvest.com (Supabase edge fn)
# because this host doesn't honor _redirects proxying. Listing them here
# establishes cross-host ownership for search engines.
# Progressive coverage: the children ramp toward the FULL catalog (every
# item, creator, and player) at ~10,000 new URLs per week, computed
# deterministically from the calendar week. Items page=2 self-populates
# once coverage passes 50k. See supabase/functions/sitemap/index.ts.
Sitemap: https://rbxinvest.com/sitemap.xml
Sitemap: https://rbxinvest.com/sitemap-static.xml
Sitemap: https://rbxinvest.com/sitemap-collections.xml
Sitemap: https://api.rbxinvest.com/functions/v1/sitemap?kind=items&page=1
Sitemap: https://api.rbxinvest.com/functions/v1/sitemap?kind=items&page=2
Sitemap: https://api.rbxinvest.com/functions/v1/sitemap?kind=items&page=3
Sitemap: https://api.rbxinvest.com/functions/v1/sitemap?kind=items&page=4
Sitemap: https://api.rbxinvest.com/functions/v1/sitemap?kind=players
Sitemap: https://api.rbxinvest.com/functions/v1/sitemap?kind=creators
Sitemap: https://api.rbxinvest.com/functions/v1/sitemap?kind=articles