# USP Demo Solutions robots.txt # Last updated: 2026-07 # Non-indexable routes (kept in sync with NOINDEX_ROUTES in prerender.ts): # /thank-you post-conversion screen, no SEO value # /demo-builder internal builder UI, not intended for discovery # # Every named crawler group explicitly disallows these routes so bots that # match a named group (and therefore bypass the wildcard) still respect the # same exclusions. # Default: allow all conventional crawlers everywhere except noindex routes. User-agent: * Allow: / Disallow: /thank-you Disallow: /thank-you/ Disallow: /demo-builder Disallow: /demo-builder/ # --- Major search engines --- User-agent: Googlebot Allow: / Disallow: /thank-you Disallow: /thank-you/ Disallow: /demo-builder Disallow: /demo-builder/ User-agent: Googlebot-Image Allow: / Disallow: /thank-you Disallow: /thank-you/ Disallow: /demo-builder Disallow: /demo-builder/ User-agent: Bingbot Allow: / Disallow: /thank-you Disallow: /thank-you/ Disallow: /demo-builder Disallow: /demo-builder/ User-agent: DuckDuckBot Allow: / Disallow: /thank-you Disallow: /thank-you/ Disallow: /demo-builder Disallow: /demo-builder/ User-agent: Applebot Allow: / Disallow: /thank-you Disallow: /thank-you/ Disallow: /demo-builder Disallow: /demo-builder/ # --- AI / LLM crawlers (training + retrieval) --- # We allow these so our content is discoverable in ChatGPT, Claude, # Perplexity, Gemini and other AI answer engines but not noindex routes. User-agent: GPTBot Allow: / Disallow: /thank-you Disallow: /thank-you/ Disallow: /demo-builder Disallow: /demo-builder/ User-agent: OAI-SearchBot Allow: / Disallow: /thank-you Disallow: /thank-you/ Disallow: /demo-builder Disallow: /demo-builder/ User-agent: ChatGPT-User Allow: / Disallow: /thank-you Disallow: /thank-you/ Disallow: /demo-builder Disallow: /demo-builder/ User-agent: ClaudeBot Allow: / Disallow: /thank-you Disallow: /thank-you/ Disallow: /demo-builder Disallow: /demo-builder/ User-agent: Claude-Web Allow: / Disallow: /thank-you Disallow: /thank-you/ Disallow: /demo-builder Disallow: /demo-builder/ User-agent: anthropic-ai Allow: / Disallow: /thank-you Disallow: /thank-you/ Disallow: /demo-builder Disallow: /demo-builder/ User-agent: PerplexityBot Allow: / Disallow: /thank-you Disallow: /thank-you/ Disallow: /demo-builder Disallow: /demo-builder/ User-agent: Perplexity-User Allow: / Disallow: /thank-you Disallow: /thank-you/ Disallow: /demo-builder Disallow: /demo-builder/ User-agent: Google-Extended Allow: / Disallow: /thank-you Disallow: /thank-you/ Disallow: /demo-builder Disallow: /demo-builder/ User-agent: Applebot-Extended Allow: / Disallow: /thank-you Disallow: /thank-you/ Disallow: /demo-builder Disallow: /demo-builder/ User-agent: CCBot Allow: / Disallow: /thank-you Disallow: /thank-you/ Disallow: /demo-builder Disallow: /demo-builder/ User-agent: meta-externalagent Allow: / Disallow: /thank-you Disallow: /thank-you/ Disallow: /demo-builder Disallow: /demo-builder/ User-agent: FacebookBot Allow: / Disallow: /thank-you Disallow: /thank-you/ Disallow: /demo-builder Disallow: /demo-builder/ User-agent: Bytespider Allow: / Disallow: /thank-you Disallow: /thank-you/ Disallow: /demo-builder Disallow: /demo-builder/ User-agent: Amazonbot Allow: / Disallow: /thank-you Disallow: /thank-you/ Disallow: /demo-builder Disallow: /demo-builder/ # --- Sitemap & LLM-friendly index --- Sitemap: https://www.usp.at/sitemap.xml # LLM-friendly content index (https://llmstxt.org/) # Categorized link list: https://www.usp.at/llms.txt # Full body text per page: https://www.usp.at/llms-full.txt # Machine-readable agent API (read-only static JSON, no auth) # Discovery manifest: https://www.usp.at/ai.json # Dataset catalog: https://www.usp.at/api/v1/index.json # The manifest is also mirrored at /.well-known/ai.json, but the static host # does not serve dot-directory paths, so the root /ai.json is canonical.