{ "project": "Verified Bots Directory", "generated_at": "2026-08-23T13:31:38.100Z", "count": 13, "bots": [ { "id": "censys", "name": "Censys Inspect", "operator": { "name": "Censys", "url": "https://censys.com" }, "description": "Censys' internet-wide scanner, which probes public IP addresses to build the host/service data behind Censys Search. Censys publishes a fixed set of scanner subnets and ASNs for opt-out purposes.", "docs": [ "https://docs.censys.com/docs/opt-out-of-data-collection" ], "category": "security", "user_agents": { "patterns": [ "CensysInspect/[\\d.]+" ], "instances": [ "Mozilla/5.0 (compatible; CensysInspect/1.1; +https://about.censys.io/)" ] }, "behavior": { "respects_robots_txt": false }, "verification": [ { "type": "static_cidrs", "cidrs": [ "66.132.159.0/24", "66.132.148.0/24", "66.132.153.0/24", "66.132.224.0/24", "66.132.186.0/24", "66.132.195.0/24", "66.132.172.0/24", "162.142.125.0/24", "167.94.138.0/24", "167.94.145.0/24", "167.94.146.0/24", "167.248.133.0/24", "199.45.154.0/24", "199.45.155.0/24", "206.168.34.0/24", "206.168.35.0/24", "2602:80d:1000:b0cc:e::/80", "2620:96:e000:b0cc:e::/80", "2602:80d:1003::/112", "2602:80d:1004::/112" ] }, { "type": "asn", "numbers": [ 398722, 398705, 398324 ] } ], "tier": 2, "tier_label": "Verifiable", "first_seen": "2026-07-08T19:13:49.571Z", "resolved_cidrs": [ "162.142.125.0/24", "167.248.133.0/24", "167.94.138.0/24", "167.94.145.0/24", "167.94.146.0/24", "199.45.154.0/24", "199.45.155.0/24", "206.168.34.0/24", "206.168.35.0/24", "2602:80d:1000:b0cc:e::/80", "2602:80d:1003::/112", "2602:80d:1004::/112", "2620:96:e000:b0cc:e::/80", "66.132.148.0/24", "66.132.153.0/24", "66.132.159.0/24", "66.132.172.0/24", "66.132.186.0/24", "66.132.195.0/24", "66.132.224.0/24" ] }, { "id": "cloudflare-custom-hostname-verification", "name": "Cloudflare Custom Hostname Verification", "operator": { "name": "Cloudflare", "url": "https://www.cloudflare.com" }, "description": "Cloudflare's custom-hostname ownership checker. Cloudflare documents that requests carrying this user agent are triggered when a customer chooses to validate a custom hostname with an HTTP ownership token, which requires fetching the token from the hostname being claimed.", "docs": [ "https://developers.cloudflare.com/fundamentals/reference/cloudflare-site-crawling/", "https://developers.cloudflare.com/cloudflare-for-platforms/cloudflare-for-saas/domain-support/hostname-validation/" ], "category": "security", "user_agents": { "patterns": [ "Cloudflare Custom Hostname Verification" ], "instances": [ "Cloudflare Custom Hostname Verification" ] }, "behavior": { "respects_robots_txt": false }, "verification": [], "tier": 3, "tier_label": "Listed only", "first_seen": "2026-07-30T16:13:28.285Z", "resolved_cidrs": [] }, { "id": "cloudflare-ssl-detector", "name": "Cloudflare SSL Detector", "operator": { "name": "Cloudflare", "url": "https://www.cloudflare.com" }, "description": "Cloudflare's SSL/TLS Recommender crawler, which fetches a customer origin over both HTTP and HTTPS to determine whether the site can be served fully over HTTPS and to recommend TLS settings.", "docs": [ "https://developers.cloudflare.com/fundamentals/reference/cloudflare-site-crawling/", "https://developers.cloudflare.com/ssl/origin-configuration/ssl-tls-recommender/" ], "category": "security", "user_agents": { "patterns": [ "Cloudflare-SSLDetector" ], "instances": [ "Cloudflare-SSLDetector" ] }, "behavior": { "respects_robots_txt": false }, "verification": [], "tier": 3, "tier_label": "Listed only", "first_seen": "2026-07-16T00:59:46.504Z", "resolved_cidrs": [] }, { "id": "cortex-xpanse", "name": "Cortex Xpanse", "operator": { "name": "Palo Alto Networks", "url": "https://www.paloaltonetworks.com" }, "description": "Palo Alto Networks' attack-surface-management scanner. It continuously scans the global internet from a published set of ranges to map its customers' internet-facing assets and discover emerging threats, and its requests carry a plain-English user agent naming the company and an opt-out contact address.", "docs": [ "https://docs-cortex.paloaltonetworks.com/r/1/Cortex-Xpanse/Scanning-activity", "https://www.paloaltonetworks.com/cortex/cortex-xpanse" ], "category": "security", "user_agents": { "patterns": [ "Expanse, a Palo Alto Networks company" ], "instances": [ "Expanse, a Palo Alto Networks company, searches across the global IPv4 space multiple times per day to identify customers' presences on the Internet. If you would like to be excluded from our scans, please send IP addresses/domains to: scaninfo@paloaltonetworks.com" ] }, "behavior": { "respects_robots_txt": false }, "verification": [ { "type": "static_cidrs", "cidrs": [ "35.203.210.0/23", "144.86.173.0/24", "147.185.132.0/23", "162.216.149.0/24", "162.216.150.0/24", "172.105.147.0/24", "198.235.24.0/24", "205.210.31.0/24", "216.25.88.0/21", "2604:a940:300:5b6::/64", "2604:a940:301:225::/64", "2604:a940:302:118::/64" ] } ], "tier": 2, "tier_label": "Verifiable", "first_seen": "2026-08-04T20:03:13.870Z", "resolved_cidrs": [ "144.86.173.0/24", "147.185.132.0/23", "162.216.149.0/24", "162.216.150.0/24", "172.105.147.0/24", "198.235.24.0/24", "205.210.31.0/24", "216.25.88.0/21", "2604:a940:300:5b6::/64", "2604:a940:301:225::/64", "2604:a940:302:118::/64", "35.203.210.0/23" ] }, { "id": "detectify", "name": "Detectify", "operator": { "name": "Detectify", "url": "https://detectify.com" }, "description": "Detectify's external attack-surface and vulnerability scanner, which probes customer-configured assets from a documented set of AWS-hosted source IPs. The operator publishes both the source IPs and the scanner's user-agent strings.", "docs": [ "https://docs.detectify.com/network-setup/scanner-ip-addresses" ], "category": "security", "user_agents": { "patterns": [ "Detectify" ], "instances": [ "Detectify", "Mozilla/5.0 (compatible; Detectify)" ] }, "behavior": { "respects_robots_txt": false }, "verification": [ { "type": "static_cidrs", "cidrs": [ "52.17.98.131/32", "52.17.9.21/32", "107.20.158.220/32", "3.234.180.95/32", "34.234.177.119/32", "13.126.5.12/32", "3.7.157.159/32", "3.7.173.162/32", "63.32.130.39/32", "54.73.182.190/32", "52.208.235.127/32" ] } ], "tier": 2, "tier_label": "Verifiable", "first_seen": "2026-07-08T19:13:49.571Z", "resolved_cidrs": [ "107.20.158.220/32", "13.126.5.12/32", "3.234.180.95/32", "3.7.157.159/32", "3.7.173.162/32", "34.234.177.119/32", "52.17.9.21/32", "52.17.98.131/32", "52.208.235.127/32", "54.73.182.190/32", "63.32.130.39/32" ] }, { "id": "internet-measurement-driftnet", "name": "Driftnet Internet Measurement", "operator": { "name": "Driftnet", "url": "https://driftnet.io" }, "description": "Driftnet's internet-measurement scanner, run from the internet-measurement.com domain, which probes publicly exposed services to give network owners an external view of their infrastructure. The operator states this traffic never attempts to log in to systems and publishes its scanner IP ranges.", "docs": [ "https://internet-measurement.com" ], "category": "security", "user_agents": { "patterns": [ "InternetMeasurement/[\\d.]+" ], "instances": [ "Mozilla/5.0 (compatible; InternetMeasurement/1.0; +https://internet-measurement.com/)" ] }, "behavior": { "respects_robots_txt": true, "robots_token": "InternetMeasurement" }, "verification": [ { "type": "static_cidrs", "cidrs": [ "87.236.176.0/24", "185.247.137.0/24", "193.163.125.0/24", "195.96.139.0/24", "45.55.151.3/32", "45.55.153.86/32", "45.55.158.168/32", "45.55.185.224/32", "45.55.186.92/32", "64.227.99.138/32", "64.227.108.146/32", "64.227.109.89/32", "64.227.110.161/32", "107.170.65.169/32", "128.199.8.140/32", "157.245.243.118/32", "157.245.245.246/32", "159.65.216.50/32", "159.65.219.252/32", "162.243.114.171/32", "162.243.116.182/32", "162.243.208.127/32", "167.99.234.119/32", "192.241.179.235/32", "2a06:4880::/29", "2604:a880:0:202a::b41:8000/124", "2604:a880:0:202a::b41:a000/124", "2604:a880:0:202a::b41:b000/124", "2604:a880:0:202a::b42:d000/124", "2604:a880:0:202a::b42:e000/124", "2604:a880:4:1d0::2fa6:a000/124", "2604:a880:4:1d0::2fa6:b000/124", "2604:a880:4:1d0::2fa6:c000/124", "2604:a880:4:1d0::2fa6:d000/124", "2604:a880:4:1d0::2fa6:e000/124", "2604:a880:400:d1::91e4:a000/124", "2604:a880:400:d1::91e4:b000/124", "2604:a880:400:d1::91e4:c000/124", "2604:a880:400:d1::91e4:d000/124", "2604:a880:400:d1::91e4:e000/124", "2604:a880:800:14::5633:8000/124", "2604:a880:800:14::5633:9000/124", "2604:a880:800:14::5633:a000/124", "2604:a880:800:14::5633:b000/124", "2604:a880:800:14::5633:c000/124" ] } ], "tier": 2, "tier_label": "Verifiable", "first_seen": "2026-07-08T19:13:49.571Z", "resolved_cidrs": [ "107.170.65.169/32", "128.199.8.140/32", "157.245.243.118/32", "157.245.245.246/32", "159.65.216.50/32", "159.65.219.252/32", "162.243.114.171/32", "162.243.116.182/32", "162.243.208.127/32", "167.99.234.119/32", "185.247.137.0/24", "192.241.179.235/32", "193.163.125.0/24", "195.96.139.0/24", "2604:a880:0:202a::b41:8000/124", "2604:a880:0:202a::b41:a000/124", "2604:a880:0:202a::b41:b000/124", "2604:a880:0:202a::b42:d000/124", "2604:a880:0:202a::b42:e000/124", "2604:a880:400:d1::91e4:a000/124", "2604:a880:400:d1::91e4:b000/124", "2604:a880:400:d1::91e4:c000/124", "2604:a880:400:d1::91e4:d000/124", "2604:a880:400:d1::91e4:e000/124", "2604:a880:4:1d0::2fa6:a000/124", "2604:a880:4:1d0::2fa6:b000/124", "2604:a880:4:1d0::2fa6:c000/124", "2604:a880:4:1d0::2fa6:d000/124", "2604:a880:4:1d0::2fa6:e000/124", "2604:a880:800:14::5633:8000/124", "2604:a880:800:14::5633:9000/124", "2604:a880:800:14::5633:a000/124", "2604:a880:800:14::5633:b000/124", "2604:a880:800:14::5633:c000/124", "2a06:4880::/29", "45.55.151.3/32", "45.55.153.86/32", "45.55.158.168/32", "45.55.185.224/32", "45.55.186.92/32", "64.227.108.146/32", "64.227.109.89/32", "64.227.110.161/32", "64.227.99.138/32", "87.236.176.0/24" ] }, { "id": "jusprog-crawler", "name": "Jugendschutzprogramm-Crawler", "operator": { "name": "JusProg e.V.", "url": "https://www.jugendschutzprogramm.de" }, "description": "Web crawler operated by JusProg e.V., a German non-profit youth-protection association, that fetches and rates web pages to build the age-classification database used by its parental-control filtering software.", "docs": [ "https://www.jugendschutzprogramm.de/en/" ], "category": "security", "user_agents": { "patterns": [ "Jugendschutzprogramm-Crawler" ], "instances": [ "Jugendschutzprogramm-Crawler; Info: http://www.jugendschutzprogramm.de" ] }, "behavior": { "respects_robots_txt": false }, "verification": [], "tier": 3, "tier_label": "Listed only", "first_seen": "2026-07-16T01:14:20.999Z", "resolved_cidrs": [] }, { "id": "leakix", "name": "LeakIX (l9explore)", "operator": { "name": "LeakIX", "url": "https://leakix.net" }, "description": "LeakIX's internet-wide recon scanner (l9explore), which probes exposed services to populate the LeakIX misconfiguration/vulnerability search engine. Its probe fleet publishes a live, per-host list of source IPs at scan.leakix.net whose hostnames resolve under scan.leakix.org, and its scanning tool sets an identifiable user-agent.", "docs": [ "https://docs.leakix.net/", "https://scan.leakix.net/" ], "category": "security", "user_agents": { "patterns": [ "l9explore/[\\d.]+" ], "instances": [ "l9explore/1.2.2" ] }, "behavior": { "respects_robots_txt": false }, "verification": [ { "type": "rdns", "masks": [ "*.scan.leakix.org" ] } ], "tier": 2, "tier_label": "Verifiable", "first_seen": "2026-07-08T19:13:49.571Z", "resolved_cidrs": [] }, { "id": "qualys-ssllabs", "name": "Qualys SSL Labs", "operator": { "name": "Qualys", "url": "https://www.qualys.com" }, "description": "Qualys' SSL Labs server test, which runs visitor-initiated and monthly (SSL Pulse) TLS-configuration assessments of public HTTPS sites. Qualys documents the assessments as slow and non-intrusive and publishes the scanner's source IP ranges in its support knowledge base.", "docs": [ "https://www.ssllabs.com/about/assessment.html", "https://success.qualys.com/support/s/article/000005823" ], "category": "security", "user_agents": { "patterns": [ "SSL Labs \\(https://www\\.ssllabs\\.com/about/assessment\\.html\\)" ], "instances": [ "SSL Labs (https://www.ssllabs.com/about/assessment.html)" ] }, "behavior": { "respects_robots_txt": false }, "verification": [ { "type": "static_cidrs", "cidrs": [ "69.67.183.0/24", "2602:fdaa:c6:2::/64", "64.39.109.20/32" ] } ], "tier": 2, "tier_label": "Verifiable", "first_seen": "2026-07-09T03:01:47.690Z", "resolved_cidrs": [ "2602:fdaa:c6:2::/64", "64.39.109.20/32", "69.67.183.0/24" ] }, { "id": "streamline3bot", "name": "Streamline3Bot", "operator": { "name": "UBT (EU) Ltd", "url": "https://www.ubtsupport.com" }, "description": "UBT's web crawler, which powers a classification service that categorises public websites by their content. It re-crawls a given site roughly once every three days. UBT publishes no IP list, and instead documents reverse-DNS verification against the ubtsupport.com domain.", "docs": [ "https://www.ubtsupport.com/legal/Streamline3Bot.php" ], "category": "security", "user_agents": { "patterns": [ "Streamline3Bot" ], "instances": [ "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1) Streamline3Bot/1.0", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; +https://www.ubtsupport.com/legal/Streamline3Bot.php) Streamline3Bot/1.0" ] }, "behavior": { "respects_robots_txt": false }, "verification": [ { "type": "rdns", "masks": [ "*.ubtsupport.com" ] } ], "tier": 2, "tier_label": "Verifiable", "first_seen": "2026-07-28T19:51:24.263Z", "resolved_cidrs": [] }, { "id": "stripe-crawler", "name": "Stripebot", "operator": { "name": "Stripe", "url": "https://stripe.com" }, "description": "Stripe's automated web crawler. Collects data from Stripe users' websites so Stripe can provide its services and comply with financial regulations. Distinct from Stripe's webhook delivery traffic.", "docs": [ "https://docs.stripe.com/stripebot-crawler" ], "category": "security", "user_agents": { "patterns": [ "Stripebot/\\d" ], "instances": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 (Stripebot/1.0; +https://docs.stripe.com/stripebot-crawler)" ] }, "behavior": { "respects_robots_txt": true, "robots_token": "Stripebot" }, "verification": [ { "type": "rdns", "masks": [ "*.crawl.stripe.com" ] } ], "tier": 2, "tier_label": "Verifiable", "first_seen": "2026-07-15T01:33:41.731Z", "resolved_cidrs": [] }, { "id": "surdotlybot", "name": "SurdotlyBot", "operator": { "name": "Sur.ly", "url": "https://sur.ly" }, "description": "Sur.ly's crawler. The operator runs a spam-fighting link-safety service and uses this bot to fetch third-party sites and build a short security profile for each one, querying metadata and favicons and taking a screenshot of the homepage. The operator states it never harvests e-mail addresses or content unrelated to security.", "docs": [ "https://sur.ly/bot.html" ], "category": "security", "user_agents": { "patterns": [ "SurdotlyBot" ], "instances": [ "Mozilla/5.0 (compatible; SurdotlyBot/1.0; +http://sur.ly/bot.html; Linux; Android 4; iPhone; CPU iPhone OS 6_0_1 like Mac OS X)" ] }, "behavior": { "respects_robots_txt": true, "robots_token": "SurdotlyBot" }, "verification": [], "tier": 3, "tier_label": "Listed only", "first_seen": "2026-08-02T14:52:18.385Z", "resolved_cidrs": [] }, { "id": "w3c-validator", "name": "W3C Markup Validator", "operator": { "name": "World Wide Web Consortium (W3C)", "url": "https://www.w3.org" }, "description": "The W3C Markup Validation Service's fetcher, used when a user submits a URL to be checked for HTML conformance. W3C documents a fixed source address for its validation services alongside the user-agent string.", "docs": [ "https://validator.w3.org/services" ], "category": "security", "user_agents": { "patterns": [ "W3C_Validator/[\\d.]+" ], "instances": [ "W3C_Validator/1.3" ] }, "behavior": { "respects_robots_txt": false }, "verification": [ { "type": "static_cidrs", "cidrs": [ "52.22.66.203/32", "2600:1f18:7d7a:2700::/56" ] } ], "tier": 2, "tier_label": "Verifiable", "first_seen": "2026-07-08T19:13:49.571Z", "resolved_cidrs": [ "2600:1f18:7d7a:2700::/56", "52.22.66.203/32" ] } ] }