# Search and answer-retrieval crawlers. These agents may help people discover
# and cite public MyIPScan pages, but they must not crawl private or temporary
# operational paths.
User-agent: Googlebot
User-agent: Bingbot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: Applebot
Allow: /
Disallow: /api/
Disallow: /_partials/
Disallow: /audit-tmp/
Disallow: /audit_full_site_launch_readiness/
Disallow: /backup/
Disallow: /backups/
Disallow: /tmp/

# AI training and broad web-corpus crawlers. MyIPScan currently allows public
# page access for visibility, while keeping private/internal paths out of scope.
# Abuse, excessive load, or content misuse should be handled with server-side
# monitoring and rate limits before changing this policy.
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: CCBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Bytespider
Allow: /
Disallow: /api/
Disallow: /_partials/
Disallow: /audit-tmp/
Disallow: /audit_full_site_launch_readiness/
Disallow: /backup/
Disallow: /backups/
Disallow: /tmp/

User-agent: *
Allow: /
Disallow: /api/
Disallow: /_partials/
Disallow: /audit-tmp/
Disallow: /audit_full_site_launch_readiness/
Disallow: /backup/
Disallow: /backups/
Disallow: /tmp/

Sitemap: https://myipscan.net/sitemap.xml