User-agent: *
Allow: /
# Public read API is crawlable (docs page + JSON endpoints) — it exists
# so that AI agents can use VetJobs as a job source. See /llms.txt.
Allow: /api/v1/
# MCP server (issue #140): read-only tools over the same inventory.
Allow: /mcp
# MCP registry domain proof (served only when MCP_REGISTRY_AUTH is set).
Allow: /.well-known/
Disallow: /admin/
Disallow: /not-even-wrong/
Disallow: /private/
Disallow: /static/admin/
Disallow: /jobs/expired/
Disallow: /404/
Disallow: /500/
Disallow: /__reload__/
# /jobs/search/ is deliberately NOT disallowed here (issue #71). The view
# sends "X-Robots-Tag: noindex, follow"; a Disallow would stop Google from
# fetching the page, so the noindex would never be read and search URLs would
# stay in the index, competing with the landing pages that should answer the
# query. noindex + crawlable is the combination that actually removes them.
Disallow: /job/*/apply/
Disallow: /jobs/job/*/apply/
Disallow: /jobs/checkout/success/
Disallow: /jobs/webhook/stripe/
Disallow: /jobs/test-tinymce/

# AI answer engines & LLM crawlers are welcome to crawl public job listings.
# (Blocking these makes the site invisible in ChatGPT, Claude, Gemini and
# Perplexity answers; jobs are public content so there is nothing to protect.)
#
# ByteDance's scraper remains blocked:
User-agent: Bytespider
Disallow: /

# Jobiglo (au.jobiglo.com) — a competitor board that re-fetched ~9,000 URLs
# in under three days from a single OVH IP (51.38.52.20), including 2,169
# `/apply/` hits. Its own robots.txt shows directives are advisory to it, so
# expect this to be ignored; it is recorded here so any later rate limit or
# block is a documented escalation rather than a surprise (2026-10-31).
#
# Note it fetches both `/jobs/job/<slug>` and the same URL with a trailing
# slash, so roughly half its requests are duplicate round trips.
User-agent: Jobiglo-Bot
Disallow: /

Sitemap: https://vetjobs.au/sitemap.xml

# Feeds (robots.txt has no directive for these — a comment is where a
# curious crawler looks):
#   RSS latest:  https://vetjobs.au/jobs/feed/rss/
#   RSS direct:  https://vetjobs.au/jobs/feed/direct/  (first-party only)
#   JSON Feed:   https://vetjobs.au/jobs/feed/json/    (JSON Feed 1.1)
#   MCP server:  https://vetjobs.au/mcp                 (read-only tools)
# /llms.txt is deliberately crawlable (no Disallow covers it).