User-Agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Allow: /api/hero-jobs
Disallow: /login
Disallow: /signup
Disallow: /api/
Disallow: /tech-icons/

# /tech-icons/ holds hundreds of decorative technology logos rendered as <img> on the technology
# and hiring pages, never in structured data, so nothing is lost by keeping crawlers off them.
# A crawler obeys only the most specific group that names it, so the rule is repeated in every
# group below, not just in "*".
User-Agent: Googlebot
User-Agent: Bingbot
Allow: /
Allow: /api/hero-jobs
Disallow: /login
Disallow: /signup
Disallow: /api/
Disallow: /tech-icons/

# AI assistants and crawlers. /brand/, /favicons/ and /illustrations/ are decorative image
# assets with no text to ingest: measured over 7 days they took 22,765 AI-crawler requests,
# 8% of the whole AI crawl budget for this site, while /blog -- where the depth actually is --
# took 3.5%. Search engines above are untouched so image search still resolves them.
User-Agent: ChatGPT-User
User-Agent: OAI-SearchBot
User-Agent: Claude-User
User-Agent: Claude-SearchBot
User-Agent: ClaudeBot
User-Agent: PerplexityBot
Allow: /
Disallow: /login
Disallow: /signup
Disallow: /brand/
Disallow: /favicons/
Disallow: /illustrations/
Disallow: /tech-icons/

User-Agent: Google-Extended
User-Agent: GPTBot
User-Agent: CCBot
User-Agent: Applebot-Extended
User-Agent: Bytespider
User-Agent: Meta-ExternalAgent
User-Agent: Amazonbot
Allow: /
Disallow: /login
Disallow: /signup
Disallow: /brand/
Disallow: /favicons/
Disallow: /illustrations/
Disallow: /tech-icons/

Sitemap: https://jobspipe.dev/sitemap.xml
Sitemap: https://docs.jobspipe.dev/sitemap.xml
schemamap: https://jobspipe.dev/schema-map.xml