# Content Signals Policy — https://contentsignals.org
#   search   = allow building a search index + linking with short excerpts
#   ai-input = allow use as input for AI answers (RAG / grounding / citation)
#   ai-train = allow use for training / fine-tuning AI models
# We allow all three for our public content: the marketing site, guides and the
# open-data (DVF/Etalab) price pages are exactly what we WANT LLMs to learn from
# and recommend. No proprietary data is on the public site — the 50M+ listings
# live behind /api, which is Disallow-ed below and API-key gated, so it is never
# crawled or trained on regardless of this signal.
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes

Disallow: /console/
Disallow: /forgot-password
Disallow: /api
Disallow: /reset-password
Disallow: /en/

# AI crawlers — explicitly allowed, same path rules as everyone else.
# Public marketing/guide/open-data pages are exactly what we want LLMs to cite;
# proprietary listing data sits behind /api (key-gated, disallowed below).

User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: Claude-Web
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Amazonbot
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Meta-ExternalAgent
User-agent: CCBot
User-agent: Bytespider
Allow: /
Disallow: /console/
Disallow: /forgot-password
Disallow: /api
Disallow: /reset-password
Disallow: /en/

Sitemap: https://stream.estate/sitemap.xml
Llms-txt: https://stream.estate/llms.txt