# Outlines are indexable on purpose. Refusing to be crawled would not make the
# records private — they are public records in public repositories, readable by
# anyone who queries the network directly — so the only thing it would change is
# whether an ordinary person can find them. /privacy says the same thing.
#
# What is disallowed below is everything that is not a page: the operational
# surface, the API, the auth flow, and the two endpoints that cost something to
# serve. Served from / by `handlers::index::robots_txt`; the copy under /static
# is the same bytes and is not the address crawlers ask for.

User-agent: *
Allow: /

# Liveness, readiness, metrics.
Disallow: /_

# JSON for the editor, and the XRPC surface. Neither is a document.
Disallow: /api/
Disallow: /xrpc

# The OAuth flow. A crawler following it burns a PAR request and gets a redirect.
Disallow: /auth

# Session-scoped pages. A crawler sees a sign-in redirect, never the content.
Disallow: /space
Disallow: /settings
Disallow: /import

# Subtree fragments: the same bullets the full page already carries, at a second
# address. Indexing both splits the ranking and shows readers the partial one.
Disallow: /*/fragment

# Server-sent events. A crawler that opens this holds the connection until it is
# dropped, against the per-IP stream cap in §7.3 — one impolite crawler is an
# outage for everyone behind the same proxy.
Disallow: /*/events

# Search is an unbounded URL space generated from a query string: the classic
# crawl trap, where every result page links to more result pages that did not
# exist until they were requested.
Disallow: /*/search