# Content Signals (contentsignals.org) — how this site's content may be used.
# search: index for search. ai-input: use in AI answers/RAG. ai-train: model training.
# NEMPulse is public CC BY data and wants to be discoverable and cited.
#
# /api/ policy: the API is for live, on-demand use — not bulk crawling. Bulk
# training crawlers are kept out of /api/ but free to crawl the rest of the
# site. User-triggered fetchers (a person asked the assistant to look something
# up) and search-engine crawlers ARE allowed into /api/ — the first so live
# lookups work, the second so they can read the noindex header (see below).
# Note: a named user-agent group fully overrides the `*` group, so each crawler
# below must repeat `Disallow: /api/` to actually be blocked from it.
#
# Search-engine crawlers are the exception: they are explicitly ALLOWED into
# /api/. Disallowing a URL does not deindex it — Google/Bing will still list a
# blocked URL they've seen referenced, with no snippet ("Indexed, though
# blocked by robots.txt": 27 such URLs in the 2026-09-08 GSC export). A
# crawler has to be able to FETCH the URL to see the `X-Robots-Tag: noindex`
# that api_stub.py's _api_noindex middleware sets on every /api/* response.
# Blocking here would hide that directive and keep the junk entries indexed.

User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /api/

# --- Search engines: whole site, /api/ included (so they can read the
#     X-Robots-Tag: noindex we serve there and drop those URLs) ---
User-agent: Googlebot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

User-agent: Bingbot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

# --- Bulk crawlers (training): site yes, /api/ no ---
User-agent: GPTBot
Allow: /
Disallow: /api/

User-agent: ClaudeBot
Allow: /
Disallow: /api/

User-agent: anthropic-ai
Allow: /
Disallow: /api/

User-agent: Google-Extended
Allow: /
Disallow: /api/

User-agent: PerplexityBot
Allow: /
Disallow: /api/

User-agent: Applebot-Extended
Allow: /
Disallow: /api/

User-agent: CCBot
Allow: /
Disallow: /api/

User-agent: cohere-ai
Allow: /
Disallow: /api/

# --- User-triggered fetchers (real-time, a person asked): /api/ allowed ---
User-agent: ChatGPT-User
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Perplexity-User
Allow: /

Sitemap: https://nempulse.com.au/sitemap.xml