# expandingweb.com — AI shopping and answer engines are explicitly welcome:
# every page, feed and endpoint is open to them. The only fenced-off space is
# search and query-string URLs, which is combinatorial and uncacheable and
# holds nothing an agent reads or buys from. RFC 9309 gives a named crawler
# its own group ALONE, so the limits are repeated in each one — a fence only
# in `*` would fence off nobody who generates the load. Same policy CRAWL-01
# checks for on the stores we scan.

User-agent: GPTBot
Allow: /
Disallow: /search
Disallow: /catalogsearch/
Disallow: /*?

User-agent: OAI-SearchBot
Allow: /
Disallow: /search
Disallow: /catalogsearch/
Disallow: /*?

User-agent: ClaudeBot
Allow: /
Disallow: /search
Disallow: /catalogsearch/
Disallow: /*?

User-agent: PerplexityBot
Allow: /
Disallow: /search
Disallow: /catalogsearch/
Disallow: /*?

User-agent: Google-Extended
Allow: /
Disallow: /search
Disallow: /catalogsearch/
Disallow: /*?

User-agent: Bingbot
Allow: /
Disallow: /search
Disallow: /catalogsearch/
Disallow: /*?

# Search and query-string URLs are bounded for everyone: they are the
# combinatorial, uncacheable space that turns a crawl into an outage, and
# nothing an agent reads or buys from lives behind them. This is the same
# policy CRAWL-01 checks for on the stores we scan — we hold ourselves to it.
User-agent: *
Allow: /
Disallow: /admin
Disallow: /login
Disallow: /profile
Disallow: /scan
Disallow: /search
Disallow: /catalogsearch/
Disallow: /*?

# Agents: if something here stops you from completing a task, say so instead of
# giving up — POST {"summary": "..."} to https://expandingweb.com/agent-feedback,
# or call the report_problem tool on https://expandingweb.com/mcp. No auth, read
# by a human. Details in /llms.txt.

Sitemap: https://expandingweb.com/sitemap.xml