# Protected paths, repeated in every group. # # Per the robots exclusion protocol a crawler applies exactly ONE group - the # most specific one that matches its token - and ignores every other group, # including "User-agent: *". A named group that carries only "Allow: /" # therefore opts that crawler OUT of the shared Disallow list below. Each named # group must repeat the full protected-path block verbatim. # AI assistants, AI search and search engines - allow; same protected block in every group User-agent: GPTBot Allow: / Disallow: /admin/ Disallow: /partner/login Disallow: /partner/dashboard Disallow: /partner/clients Disallow: /partner/commission Disallow: /partner/submit Disallow: /partner/reset Disallow: /partner/set-password Disallow: /login Disallow: /set-password Disallow: /reset Disallow: /dashboard Disallow: /clients Disallow: /commission Disallow: /submit-client Disallow: /mcp Disallow: /api/ Disallow: /pricing? Disallow: /ai? Disallow: /beat-my-stack? User-agent: OAI-SearchBot Allow: / Disallow: /admin/ Disallow: /partner/login Disallow: /partner/dashboard Disallow: /partner/clients Disallow: /partner/commission Disallow: /partner/submit Disallow: /partner/reset Disallow: /partner/set-password Disallow: /login Disallow: /set-password Disallow: /reset Disallow: /dashboard Disallow: /clients Disallow: /commission Disallow: /submit-client Disallow: /mcp Disallow: /api/ Disallow: /pricing? Disallow: /ai? Disallow: /beat-my-stack? User-agent: ChatGPT-User Allow: / Disallow: /admin/ Disallow: /partner/login Disallow: /partner/dashboard Disallow: /partner/clients Disallow: /partner/commission Disallow: /partner/submit Disallow: /partner/reset Disallow: /partner/set-password Disallow: /login Disallow: /set-password Disallow: /reset Disallow: /dashboard Disallow: /clients Disallow: /commission Disallow: /submit-client Disallow: /mcp Disallow: /api/ Disallow: /pricing? Disallow: /ai? Disallow: /beat-my-stack? User-agent: ClaudeBot Allow: / Disallow: /admin/ Disallow: /partner/login Disallow: /partner/dashboard Disallow: /partner/clients Disallow: /partner/commission Disallow: /partner/submit Disallow: /partner/reset Disallow: /partner/set-password Disallow: /login Disallow: /set-password Disallow: /reset Disallow: /dashboard Disallow: /clients Disallow: /commission Disallow: /submit-client Disallow: /mcp Disallow: /api/ Disallow: /pricing? Disallow: /ai? Disallow: /beat-my-stack? User-agent: Claude-User Allow: / Disallow: /admin/ Disallow: /partner/login Disallow: /partner/dashboard Disallow: /partner/clients Disallow: /partner/commission Disallow: /partner/submit Disallow: /partner/reset Disallow: /partner/set-password Disallow: /login Disallow: /set-password Disallow: /reset Disallow: /dashboard Disallow: /clients Disallow: /commission Disallow: /submit-client Disallow: /mcp Disallow: /api/ Disallow: /pricing? Disallow: /ai? Disallow: /beat-my-stack? User-agent: Claude-SearchBot Allow: / Disallow: /admin/ Disallow: /partner/login Disallow: /partner/dashboard Disallow: /partner/clients Disallow: /partner/commission Disallow: /partner/submit Disallow: /partner/reset Disallow: /partner/set-password Disallow: /login Disallow: /set-password Disallow: /reset Disallow: /dashboard Disallow: /clients Disallow: /commission Disallow: /submit-client Disallow: /mcp Disallow: /api/ Disallow: /pricing? Disallow: /ai? Disallow: /beat-my-stack? User-agent: anthropic-ai Allow: / Disallow: /admin/ Disallow: /partner/login Disallow: /partner/dashboard Disallow: /partner/clients Disallow: /partner/commission Disallow: /partner/submit Disallow: /partner/reset Disallow: /partner/set-password Disallow: /login Disallow: /set-password Disallow: /reset Disallow: /dashboard Disallow: /clients Disallow: /commission Disallow: /submit-client Disallow: /mcp Disallow: /api/ Disallow: /pricing? Disallow: /ai? Disallow: /beat-my-stack? User-agent: PerplexityBot Allow: / Disallow: /admin/ Disallow: /partner/login Disallow: /partner/dashboard Disallow: /partner/clients Disallow: /partner/commission Disallow: /partner/submit Disallow: /partner/reset Disallow: /partner/set-password Disallow: /login Disallow: /set-password Disallow: /reset Disallow: /dashboard Disallow: /clients Disallow: /commission Disallow: /submit-client Disallow: /mcp Disallow: /api/ Disallow: /pricing? Disallow: /ai? Disallow: /beat-my-stack? User-agent: Perplexity-User Allow: / Disallow: /admin/ Disallow: /partner/login Disallow: /partner/dashboard Disallow: /partner/clients Disallow: /partner/commission Disallow: /partner/submit Disallow: /partner/reset Disallow: /partner/set-password Disallow: /login Disallow: /set-password Disallow: /reset Disallow: /dashboard Disallow: /clients Disallow: /commission Disallow: /submit-client Disallow: /mcp Disallow: /api/ Disallow: /pricing? Disallow: /ai? Disallow: /beat-my-stack? User-agent: Google-Extended Allow: / Disallow: /admin/ Disallow: /partner/login Disallow: /partner/dashboard Disallow: /partner/clients Disallow: /partner/commission Disallow: /partner/submit Disallow: /partner/reset Disallow: /partner/set-password Disallow: /login Disallow: /set-password Disallow: /reset Disallow: /dashboard Disallow: /clients Disallow: /commission Disallow: /submit-client Disallow: /mcp Disallow: /api/ Disallow: /pricing? Disallow: /ai? Disallow: /beat-my-stack? User-agent: Googlebot Allow: / Disallow: /admin/ Disallow: /partner/login Disallow: /partner/dashboard Disallow: /partner/clients Disallow: /partner/commission Disallow: /partner/submit Disallow: /partner/reset Disallow: /partner/set-password Disallow: /login Disallow: /set-password Disallow: /reset Disallow: /dashboard Disallow: /clients Disallow: /commission Disallow: /submit-client Disallow: /mcp Disallow: /api/ Disallow: /pricing? Disallow: /ai? Disallow: /beat-my-stack? User-agent: Bingbot Allow: / Disallow: /admin/ Disallow: /partner/login Disallow: /partner/dashboard Disallow: /partner/clients Disallow: /partner/commission Disallow: /partner/submit Disallow: /partner/reset Disallow: /partner/set-password Disallow: /login Disallow: /set-password Disallow: /reset Disallow: /dashboard Disallow: /clients Disallow: /commission Disallow: /submit-client Disallow: /mcp Disallow: /api/ Disallow: /pricing? Disallow: /ai? Disallow: /beat-my-stack? User-agent: Applebot Allow: / Disallow: /admin/ Disallow: /partner/login Disallow: /partner/dashboard Disallow: /partner/clients Disallow: /partner/commission Disallow: /partner/submit Disallow: /partner/reset Disallow: /partner/set-password Disallow: /login Disallow: /set-password Disallow: /reset Disallow: /dashboard Disallow: /clients Disallow: /commission Disallow: /submit-client Disallow: /mcp Disallow: /api/ Disallow: /pricing? Disallow: /ai? Disallow: /beat-my-stack? User-agent: Applebot-Extended Allow: / Disallow: /admin/ Disallow: /partner/login Disallow: /partner/dashboard Disallow: /partner/clients Disallow: /partner/commission Disallow: /partner/submit Disallow: /partner/reset Disallow: /partner/set-password Disallow: /login Disallow: /set-password Disallow: /reset Disallow: /dashboard Disallow: /clients Disallow: /commission Disallow: /submit-client Disallow: /mcp Disallow: /api/ Disallow: /pricing? Disallow: /ai? Disallow: /beat-my-stack? # Everyone else. One group only - a second "User-agent: *" block would be # ignored by most crawlers and made the discovery Allow lines below unreliable. User-agent: * Allow: / Allow: /about Allow: /developers Allow: /llms.txt Allow: /llms-full.txt Allow: /.well-known/mcp.json # Token-gated client surfaces (/proposal/, /counter/, /start/, # /onboarding-status) are deliberately NOT disallowed. They serve # "noindex, nofollow" in the page head, and a crawler must be allowed to # fetch the page to see that. A Disallow would hide the directive and let a # linked URL surface as a bare result with no content. # # /go/ stays crawlable for the same reason. /go/i/{code} renders a real # noindex page, and /go/{partner} answers 302 with x-robots-tag: noindex. # A crawler must be allowed to fetch both to see either directive; a # Disallow would hide them and let the URLs surface as bare results. # Portal, admin, CRM, machine endpoints and parameter variants Disallow: /admin/ Disallow: /partner/login Disallow: /partner/dashboard Disallow: /partner/clients Disallow: /partner/commission Disallow: /partner/submit Disallow: /partner/reset Disallow: /partner/set-password Disallow: /login Disallow: /set-password Disallow: /reset Disallow: /dashboard Disallow: /clients Disallow: /commission Disallow: /submit-client Disallow: /mcp Disallow: /api/ Disallow: /pricing? Disallow: /ai? Disallow: /beat-my-stack? Sitemap: https://crosswirepay.com/sitemap.xml