User-agent: *
Allow: /api/auth.php
Allow: /api/beacon.php
Allow: /
Disallow: /api/
Disallow: /cache/
Disallow: /includes/
Disallow: /iadmin/
Disallow: /static/favicons/
Disallow: /recent.php
# One page per IP address is an unbounded URL space, and every one of those
# pages fans out into 47 external RBL lookups. Nothing to index, real cost to
# serve.
Disallow: /ip-blacklist-checker.php
Disallow: /ip-blacklist-checker/

# Block AI scrapers that give zero traffic back
User-agent: Bytespider
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: BitSightBot
Disallow: /

# Meta's AI-training crawlers. Not the same thing as facebookexternalhit, which
# unfurls shared links and stays allowed.
User-agent: meta-externalagent
Disallow: /

User-agent: meta-externalfetcher
Disallow: /

User-agent: FacebookBot
Disallow: /

Sitemap: https://ipwhois.net/sitemap.xml
Sitemap: https://ipwhois.net/sitemap-is-it-down.xml