{ "project": "Verified Bots Directory", "generated_at": "2026-08-23T13:31:38.100Z", "count": 51, "bots": [ { "id": "ahrefs-siteaudit", "name": "AhrefsSiteAudit", "operator": { "name": "Ahrefs", "url": "https://ahrefs.com" }, "description": "Ahrefs' on-demand site-auditing crawler, distinct from AhrefsBot, used when Ahrefs customers run the Site Audit tool against their own or a competitor's domain. Shares Ahrefs' published IP-range feed.", "docs": [ "https://ahrefs.com/robot/site-audit" ], "category": "seo", "user_agents": { "patterns": [ "AhrefsSiteAudit/\\d" ], "instances": [ "Mozilla/5.0 (compatible; AhrefsSiteAudit/6.1; +http://ahrefs.com/robot/site-audit)" ] }, "behavior": { "respects_robots_txt": true, "robots_token": "AhrefsSiteAudit" }, "verification": [ { "type": "cidr_feed", "url": "https://api.ahrefs.com/v3/public/crawler-ip-ranges", "format": "prefixes" } ], "tier": 1, "tier_label": "Fully verifiable", "first_seen": "2026-07-08T19:13:49.571Z", "resolved_cidrs": [ "142.44.220.0/24", "142.44.225.0/24", "142.44.228.0/24", "142.44.233.0/24", "148.113.128.0/24", "148.113.130.0/24", "15.235.27.0/24", "15.235.96.0/24", "15.235.98.0/24", "167.114.139.0/24", "168.100.149.0/24", "176.31.139.0/27", "198.244.168.0/24", "198.244.183.0/24", "198.244.186.193/32", "198.244.186.194/31", "198.244.186.196/30", "198.244.186.200/31", "198.244.186.202/32", "198.244.226.0/24", "198.244.240.0/24", "198.244.242.0/24", "202.8.40.0/22", "202.94.84.110/31", "202.94.84.112/31", "37.59.204.128/27", "5.39.1.224/27", "5.39.109.160/27", "51.161.37.0/24", "51.161.65.0/24", "51.195.183.0/24", "51.195.215.0/24", "51.195.244.0/24", "51.222.168.0/24", "51.222.253.0/26", "51.222.95.0/24", "51.68.247.192/27", "51.75.236.128/27", "51.81.103.240/28", "51.81.104.32/28", "51.81.105.16/28", "51.81.110.16/28", "51.81.110.48/28", "51.81.110.80/28", "51.81.111.0/28", "51.81.111.112/28", "51.81.111.128/28", "51.81.111.64/28", "51.81.163.16/28", "51.81.163.48/28", "51.81.163.64/28", "51.81.164.144/28", "51.81.164.192/28", "51.81.165.208/28", "51.81.165.80/28", "51.81.169.112/28", "51.81.169.192/28", "51.81.170.0/28", "51.89.129.0/24", "51.89.38.128/28", "51.89.39.48/28", "51.89.59.32/28", "51.89.69.32/28", "51.89.69.96/28", "51.89.71.144/28", "51.89.71.80/28", "51.89.72.0/28", "51.89.74.192/28", "51.89.83.48/28", "54.36.148.0/23", "54.37.118.64/27", "54.38.147.0/24", "54.39.0.0/24", "54.39.136.0/24", "54.39.203.0/24", "54.39.210.0/24", "54.39.6.0/24", "54.39.89.0/24", "92.222.104.192/27", "92.222.108.96/27", "94.23.188.192/27" ] }, { "id": "ahrefsbot", "name": "AhrefsBot", "operator": { "name": "Ahrefs", "url": "https://ahrefs.com" }, "description": "Ahrefs' primary web crawler, powering the backlink and keyword database behind the Ahrefs SEO platform and the Yep search engine. Crawls from publicly published IP ranges with a matching reverse-DNS suffix.", "docs": [ "https://ahrefs.com/robot" ], "category": "seo", "user_agents": { "patterns": [ "AhrefsBot/\\d" ], "instances": [ "Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)" ] }, "behavior": { "respects_robots_txt": true, "robots_token": "AhrefsBot" }, "verification": [ { "type": "cidr_feed", "url": "https://api.ahrefs.com/v3/public/crawler-ip-ranges", "format": "prefixes" } ], "tier": 1, "tier_label": "Fully verifiable", "first_seen": "2026-07-08T19:13:49.571Z", "resolved_cidrs": [ "142.44.220.0/24", "142.44.225.0/24", "142.44.228.0/24", "142.44.233.0/24", "148.113.128.0/24", "148.113.130.0/24", "15.235.27.0/24", "15.235.96.0/24", "15.235.98.0/24", "167.114.139.0/24", "168.100.149.0/24", "176.31.139.0/27", "198.244.168.0/24", "198.244.183.0/24", "198.244.186.193/32", "198.244.186.194/31", "198.244.186.196/30", "198.244.186.200/31", "198.244.186.202/32", "198.244.226.0/24", "198.244.240.0/24", "198.244.242.0/24", "202.8.40.0/22", "202.94.84.110/31", "202.94.84.112/31", "37.59.204.128/27", "5.39.1.224/27", "5.39.109.160/27", "51.161.37.0/24", "51.161.65.0/24", "51.195.183.0/24", "51.195.215.0/24", "51.195.244.0/24", "51.222.168.0/24", "51.222.253.0/26", "51.222.95.0/24", "51.68.247.192/27", "51.75.236.128/27", "51.81.103.240/28", "51.81.104.32/28", "51.81.105.16/28", "51.81.110.16/28", "51.81.110.48/28", "51.81.110.80/28", "51.81.111.0/28", "51.81.111.112/28", "51.81.111.128/28", "51.81.111.64/28", "51.81.163.16/28", "51.81.163.48/28", "51.81.163.64/28", "51.81.164.144/28", "51.81.164.192/28", "51.81.165.208/28", "51.81.165.80/28", "51.81.169.112/28", "51.81.169.192/28", "51.81.170.0/28", "51.89.129.0/24", "51.89.38.128/28", "51.89.39.48/28", "51.89.59.32/28", "51.89.69.32/28", "51.89.69.96/28", "51.89.71.144/28", "51.89.71.80/28", "51.89.72.0/28", "51.89.74.192/28", "51.89.83.48/28", "54.36.148.0/23", "54.37.118.64/27", "54.38.147.0/24", "54.39.0.0/24", "54.39.136.0/24", "54.39.203.0/24", "54.39.210.0/24", "54.39.6.0/24", "54.39.89.0/24", "92.222.104.192/27", "92.222.108.96/27", "94.23.188.192/27" ] }, { "id": "audigent-adbot", "name": "AudigentAdBot", "operator": { "name": "Audigent", "url": "https://audigent.com" }, "description": "Audigent's advertising crawler. The operator documents that it collects only the metadata in the header of an HTML page and does not scrape page body content, and that it must be named explicitly in robots.txt because a wildcard rule does not block it.", "docs": [ "https://audigent.com/bot.html" ], "category": "seo", "user_agents": { "patterns": [ "AudigentAdBot" ], "instances": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; AudigentAdBot; +https://audigent.com/bot.html) Chrome/W.X.Y.Z Safari/537.36" ] }, "behavior": { "respects_robots_txt": true, "robots_token": "AudigentAdBot" }, "verification": [], "tier": 3, "tier_label": "Listed only", "first_seen": "2026-08-23T13:20:46.873Z", "resolved_cidrs": [] }, { "id": "audisto", "name": "Audisto Crawler", "operator": { "name": "Audisto GmbH", "url": "https://audisto.com" }, "description": "Crawler for Audisto's hosted technical SEO and site-audit platform, fetching pages of the sites its customers analyse. Audisto publishes its crawler addresses as JSON and documents reverse-DNS verification.", "docs": [ "https://audisto.com/help/crawler/bot/" ], "category": "seo", "user_agents": { "patterns": [ "Audisto Crawler" ], "instances": [ "Audisto Crawler (mobile; +https://audisto.com/bot)", "Audisto Crawler (desktop; +https://audisto.com/bot)", "Audisto Crawler (desktop; essential; +https://audisto.com/bot)" ] }, "behavior": { "respects_robots_txt": true, "robots_token": "audisto" }, "verification": [ { "type": "cidr_feed", "url": "https://audisto.com/ips.json", "format": "github_meta", "selector": "ipv4" }, { "type": "cidr_feed", "url": "https://audisto.com/ips.json", "format": "github_meta", "selector": "ipv6" }, { "type": "rdns", "masks": [ "*.crawler.audisto.com" ] } ], "tier": 1, "tier_label": "Fully verifiable", "first_seen": "2026-07-22T13:35:18.584Z", "resolved_cidrs": [ "116.202.161.186/32", "138.201.254.123/32", "142.132.180.252/32", "148.251.68.22/32", "162.55.235.248/32", "167.235.32.214/32", "168.119.181.43/32", "168.119.245.51/32", "168.119.53.206/32", "176.9.73.44/32", "195.201.226.139/32", "195.201.226.141/32", "195.201.23.108/32", "195.201.233.95/32", "195.201.25.108/32", "195.201.25.74/32", "2a01:4f8:1c1e:8943::1/128", "2a01:4f8:1c1e:8944::1/128", "2a01:4f8:1c1e:91fd::1/128", "2a01:4f8:1c1e:9227::1/128", "2a01:4f8:1c1e:9228::1/128", "2a01:4f8:c010:be17::1/128", "2a01:4f8:c010:be18::1/128", "2a01:4f8:c010:be19::1/128", "2a01:4f8:c010:be34::1/128", "2a01:4f8:c010:be35::1/128", "46.4.40.37/32", "46.4.65.48/32", "5.9.120.199/32", "5.9.152.57/32" ] }, { "id": "barkrowler", "name": "Barkrowler", "operator": { "name": "Babbar", "url": "https://www.babbar.tech" }, "description": "Barkrowler is the web crawler operated by Babbar (formerly Exensa). It builds and updates Babbar's graph of the web, which powers the company's SEO and link-analysis tools, and applies a politeness delay between requests.", "docs": [ "https://www.babbar.tech/crawler" ], "category": "seo", "user_agents": { "patterns": [ "Bark[rR]owler" ], "instances": [ "Barkrowler/0.7 (+http://www.exensa.com/crawl)", "BarkRowler/0.7 (+http://www.exensa.com/crawling)" ] }, "behavior": { "respects_robots_txt": true, "robots_token": "barkrowler" }, "verification": [ { "type": "cidr_feed", "url": "https://www.babbar.tech/barkrowler-ip-ranges.json", "format": "prefixes" } ], "tier": 1, "tier_label": "Fully verifiable", "first_seen": "2026-07-16T01:14:20.999Z", "resolved_cidrs": [ "154.54.249.0/24", "217.113.194.0/24" ] }, { "id": "bombora-crawler", "name": "BomboraBot", "operator": { "name": "Bombora, Inc.", "url": "https://bombora.com" }, "description": "BomboraBot is Bombora's web crawler. It classifies the content and topics of web pages that carry Bombora's tags so the company can model B2B purchase intent, visiting each tagged page at most once every 30 days.", "docs": [ "https://www.bombora.com/bot" ], "category": "seo", "user_agents": { "patterns": [ "BomboraBot" ], "instances": [ "Mozilla/5.0 (compatible; BomboraBot/1.0; +http://www.bombora.com/bot)" ] }, "behavior": { "respects_robots_txt": false }, "verification": [], "tier": 3, "tier_label": "Listed only", "first_seen": "2026-07-16T01:14:20.999Z", "resolved_cidrs": [] }, { "id": "botify", "name": "Botify", "operator": { "name": "Botify", "url": "https://www.botify.com" }, "description": "Botify's SEO crawler, used by its SiteCrawler product to analyze enterprise websites for search-indexing insights. Botify does not publish a fixed IP range or CIDR list for site owners to allowlist.", "docs": [ "https://support.botify.com/en/articles/9108549-crawling-with-a-custom-user-agent" ], "category": "seo", "user_agents": { "patterns": [ "compatible; botify" ], "instances": [ "Mozilla/5.0 (compatible; botify; http://botify.com)" ] }, "behavior": { "respects_robots_txt": true, "robots_token": "Botify" }, "verification": [], "tier": 3, "tier_label": "Listed only", "first_seen": "2026-07-08T19:13:49.571Z", "resolved_cidrs": [] }, { "id": "brightbot", "name": "Brightbot", "operator": { "name": "Bright Data", "url": "https://brightdata.com" }, "description": "Bright Data's data-collection crawler, documented as the main collection pipeline for its products, with a 24-hour cache layer to avoid re-downloading the same page. The operator states Brightbot is deliberately transparent — a unique user agent plus a single published source subnet — so its traffic can be separated from user traffic. Its documented control mechanism is Bright Data's own collectors.txt file and Web Master console; the page states no robots.txt behaviour.", "docs": [ "https://brightdata.com/brightbot" ], "category": "seo", "user_agents": { "patterns": [ "Brightbot \\d" ], "instances": [ "Brightbot 1.0" ] }, "behavior": { "respects_robots_txt": false }, "verification": [ { "type": "static_cidrs", "cidrs": [ "82.97.199.0/24" ] } ], "tier": 2, "tier_label": "Verifiable", "first_seen": "2026-08-23T13:23:09.318Z", "resolved_cidrs": [ "82.97.199.0/24" ] }, { "id": "builtwith", "name": "BuiltWith", "operator": { "name": "BuiltWith Pty Ltd", "url": "https://builtwith.com" }, "description": "Crawler for BuiltWith's technology-profiling service, which visits sites and analyses publicly visible markup to determine which web technologies they use. BuiltWith publishes no IP ranges, so requests cannot be verified.", "docs": [ "https://builtwith.com/biup" ], "category": "seo", "user_agents": { "patterns": [ "BuiltWith/[\\d.]+", "BW/[\\d.]+" ], "instances": [ "BuiltWith/1.4; rb.gy/xprgqj", "BW/1.3; rb.gy/qyzae5" ] }, "behavior": { "respects_robots_txt": true, "robots_token": "BuiltWith" }, "verification": [], "tier": 3, "tier_label": "Listed only", "first_seen": "2026-07-18T15:23:43.605Z", "resolved_cidrs": [] }, { "id": "caliperbot", "name": "Caliperbot", "operator": { "name": "Conductor", "url": "https://www.conductor.com" }, "description": "Conductor's single web crawler. It reads the HTML of pages on sites its customers track, recording on-page elements such as title tags, header tags and other metadata for Conductor's SEO and search-visibility reporting. Conductor publishes the address range it crawls from and will lower the crawl rate on request.", "docs": [ "https://www.conductor.com/caliperbot", "https://www.conductor.com/docs/intelligence/conductors-data-collection-and-crawling-faqs/" ], "category": "seo", "user_agents": { "patterns": [ "Caliperbot/\\d" ], "instances": [ "Caliperbot/1.0 (+http://www.conductor.com/caliperbot)", "Caliperbot/1.0 (+https://www.conductor.com/caliperbot)" ] }, "behavior": { "respects_robots_txt": false }, "verification": [ { "type": "static_cidrs", "cidrs": [ "162.246.176.0/21" ] } ], "tier": 2, "tier_label": "Verifiable", "first_seen": "2026-07-24T13:19:52.933Z", "resolved_cidrs": [ "162.246.176.0/21" ] }, { "id": "cincraw", "name": "Cincraw", "operator": { "name": "CINC Corp. (株式会社CINC)", "url": "https://www.cinc-j.co.jp" }, "description": "The web crawler operated by CINC, a Japanese data-solutions company, to collect the page data behind its marketing and SEO analytics products. Its documented policy is to fetch page body content, header and HTTP status information and the JS/CSS needed to render a page, then store a rendered screen capture. CINC states that it does not follow advertising links, deletes all cookies between requests, and does not load analytics or ad-measurement tags. No robots.txt policy and no IP ranges are published.", "docs": [ "https://cincrawdata.net/bot/" ], "category": "seo", "user_agents": { "patterns": [ "Cincraw/[\\d.]+" ], "instances": [ "Mozilla/5.0 (compatible; Cincraw/1.0; +http://cincrawdata.net/bot/)" ] }, "behavior": { "respects_robots_txt": false }, "verification": [], "tier": 3, "tier_label": "Listed only", "first_seen": "2026-07-30T16:13:28.285Z", "resolved_cidrs": [] }, { "id": "claritybot", "name": "Claritybot", "operator": { "name": "seoClarity", "url": "https://www.seoclarity.net" }, "description": "seoClarity's page and site audit crawler. Crawls are triggered on demand by seoClarity clients to analyse pages for technical and content issues, and clients can also schedule daily managed page crawls. The operator documents that it obeys robots.txt and Crawl-delay, and that its addresses are dynamic.", "docs": [ "https://www.seoclarity.net/bot.html" ], "category": "seo", "user_agents": { "patterns": [ "ClarityBot/\\d" ], "instances": [ "Mozilla/5.0 (compatible; ClarityBot/9.0; +https://www.seoclarity.net/bot.html)", "Mozilla/5.0 (Linux; Android 9; SM-G960F Build/PPR1.180610.011; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/74.0.3729.157 Mobile Safari/537.36 (compatible; ClarityBot/9.0; +https://www.seoclarity.net/bot.html)" ] }, "behavior": { "respects_robots_txt": true, "robots_token": "claritybot" }, "verification": [], "tier": 3, "tier_label": "Listed only", "first_seen": "2026-08-23T13:20:46.873Z", "resolved_cidrs": [] }, { "id": "cocolyzebot", "name": "Cocolyzebot", "operator": { "name": "Cocolyze", "url": "https://www.cocolyze.com" }, "description": "Crawler for Cocolyze's SEO analysis platform, fetching pages of sites its users analyse. Cocolyze publishes no IP ranges, so requests cannot be verified beyond the user agent.", "docs": [ "https://www.cocolyze.com/en/bot" ], "category": "seo", "user_agents": { "patterns": [ "Cocolyzebot/\\d" ], "instances": [ "Mozilla/5.0 (compatible; Cocolyzebot/1.0; https://cocolyze.com/bot)" ] }, "behavior": { "respects_robots_txt": true, "robots_token": "cocolyzebot" }, "verification": [], "tier": 3, "tier_label": "Listed only", "first_seen": "2026-07-26T13:26:40.725Z", "resolved_cidrs": [] }, { "id": "cognitiveseo-crawler", "name": "cognitiveSEO", "operator": { "name": "cognitiveSEO", "url": "https://cognitiveseo.com" }, "description": "James BOT is the web crawler operated by cognitiveSEO, an SEO toolset. It crawls the web and analyzes links to power the backlink and SEO analysis offered by the cognitiveSEO platform.", "docs": [ "https://cognitiveseo.com/bot.html" ], "category": "seo", "user_agents": { "patterns": [ "James BOT" ], "instances": [ "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6 - James BOT - WebCrawler http://cognitiveseo.com/bot.html" ] }, "behavior": { "respects_robots_txt": true, "robots_token": "JamesBOT" }, "verification": [], "tier": 3, "tier_label": "Listed only", "first_seen": "2026-07-16T01:14:20.999Z", "resolved_cidrs": [] }, { "id": "criteo", "name": "CriteoBot", "operator": { "name": "Criteo", "url": "https://www.criteo.com" }, "description": "Criteo's advertising crawler. It fetches merchant and publisher pages to extract product and content data used for Criteo's commerce and retargeting ads, and respects robots.txt and crawl-delay directives.", "docs": [ "https://www.criteo.com/criteo-crawler/" ], "category": "seo", "user_agents": { "patterns": [ "CriteoBot" ], "instances": [ "CriteoBot/0.1 (+https://www.criteo.com/criteo-crawler/)" ] }, "behavior": { "respects_robots_txt": true, "robots_token": "CriteoBot" }, "verification": [ { "type": "asn", "numbers": [ 19750, 44788, 55569 ] }, { "type": "static_cidrs", "cidrs": [ "74.119.116.0/22", "91.199.242.0/24", "91.212.98.0/24", "116.213.20.0/22", "177.73.128.0/21", "178.250.0.0/21", "182.161.72.0/22", "185.235.84.0/22", "199.204.168.0/22", "2406:2600::/32", "2620:100:a000::/44", "2a02:2638::/32" ] } ], "tier": 2, "tier_label": "Verifiable", "first_seen": "2026-07-16T13:18:54.422Z", "resolved_cidrs": [ "116.213.20.0/22", "177.73.128.0/21", "178.250.0.0/21", "182.161.72.0/22", "185.235.84.0/22", "199.204.168.0/22", "2406:2600::/32", "2620:100:a000::/44", "2a02:2638::/32", "74.119.116.0/22", "91.199.242.0/24", "91.212.98.0/24" ] }, { "id": "dataforseo", "name": "DataForSeoBot", "operator": { "name": "DataForSEO", "url": "https://dataforseo.com" }, "description": "DataForSEO's crawler. It fetches pages to build the backlink and SEO datasets that power DataForSEO's marketing-data APIs, and honours robots.txt and crawl-delay directives.", "docs": [ "https://dataforseo.com/dataforseo-bot" ], "category": "seo", "user_agents": { "patterns": [ "DataForSeoBot" ], "instances": [ "Mozilla/5.0 (compatible; DataForSeoBot; +https://dataforseo.com/dataforseo-bot)" ] }, "behavior": { "respects_robots_txt": true, "robots_token": "DataForSeoBot" }, "verification": [ { "type": "static_cidrs", "cidrs": [ "136.243.220.208/29", "136.243.228.176/29", "136.243.228.192/29", "2a01:4f8:2b03:38b::/64", "2a01:4f8:2b03:38c::/64", "2a01:4f8:2b03:38d::/64" ] }, { "type": "rdns", "masks": [ "*.dataforseo.com" ] } ], "tier": 2, "tier_label": "Verifiable", "first_seen": "2026-07-16T13:18:54.422Z", "resolved_cidrs": [ "136.243.220.208/29", "136.243.228.176/29", "136.243.228.192/29", "2a01:4f8:2b03:38b::/64", "2a01:4f8:2b03:38c::/64", "2a01:4f8:2b03:38d::/64" ] }, { "id": "dataprovider", "name": "Dataproviderbot", "operator": { "name": "Dataprovider.com", "url": "https://www.dataprovider.com" }, "description": "Dataprovider.com's in-house crawler. It indexes more than 400 million domains each month and structures what it finds into the company's web dataset (business information, technology detection, classifications and risk signals). The operator documents that it follows the robot exclusion protocol and that its crawlers can be identified by a reverse DNS lookup.", "docs": [ "https://www.dataprovider.com/crawler/" ], "category": "seo", "user_agents": { "patterns": [ "Dataprovider\\.com" ], "instances": [ "Mozilla/5.0 (compatible; Dataprovider.com)" ] }, "behavior": { "respects_robots_txt": true, "robots_token": "dataprovider" }, "verification": [ { "type": "rdns", "masks": [ "*.dataproviderbot.com" ] } ], "tier": 2, "tier_label": "Verifiable", "first_seen": "2026-08-02T14:52:18.385Z", "resolved_cidrs": [] }, { "id": "domcopbot", "name": "DomCopBot", "operator": { "name": "DomCop", "url": "https://www.domcop.com" }, "description": "DomCop's availability crawler, used by the domain-research service of the same name. The operator documents that it accesses only the robots.txt file, once per domain, and uses that request to establish whether a website is live on the domain.", "docs": [ "https://www.domcop.com/bot" ], "category": "seo", "user_agents": { "patterns": [ "DomCopBot" ], "instances": [ "DomCopBot (https://www.domcop.com/bot)" ] }, "behavior": { "respects_robots_txt": true, "robots_token": "DomCopBot" }, "verification": [], "tier": 3, "tier_label": "Listed only", "first_seen": "2026-08-23T13:20:46.873Z", "resolved_cidrs": [] }, { "id": "dotbot", "name": "DotBot", "operator": { "name": "Moz", "url": "https://moz.com" }, "description": "Moz's general-purpose web crawler, distinct from rogerbot, that gathers link data powering the Moz Link Index and Link Explorer. Moz's own help pages document no fixed IP range for it.", "docs": [ "https://moz.com/help/moz-procedures/crawlers/dotbot" ], "category": "seo", "user_agents": { "patterns": [ "DotBot/\\d" ], "instances": [ "Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, help@moz.com)" ] }, "behavior": { "respects_robots_txt": true, "robots_token": "dotbot" }, "verification": [], "tier": 3, "tier_label": "Listed only", "first_seen": "2026-07-08T19:13:49.571Z", "resolved_cidrs": [] }, { "id": "dragonbot", "name": "Dragonbot", "operator": { "name": "Dragon Metrics", "url": "https://www.dragonmetrics.com" }, "description": "Dragon Metrics' SEO crawler, which collects data for the platform's Site Audit and Site Explorer features. Its operator documents that it respects robots.txt using Google's open-source parser, and that it crawls from dynamic IP addresses so it can only be identified by user agent.", "docs": [ "https://help.dragonmetrics.com/en/articles/4615406-what-is-the-user-agent-string-for-dragon-metrics-crawler-dragonbot", "https://help.dragonmetrics.com/en/articles/4316806-how-to-block-dragonbot-dragon-metrics-crawler-by-robots-txt", "https://help.dragonmetrics.com/en/articles/4615389-what-is-the-ip-address-of-the-dragon-metrics-crawler-dragonbot" ], "category": "seo", "user_agents": { "patterns": [ "Dragonbot; http://www\\.dragonmetrics\\.com" ], "instances": [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36;Dragonbot; http://www.dragonmetrics.com" ] }, "behavior": { "respects_robots_txt": true, "robots_token": "Dragonbot" }, "verification": [], "tier": 3, "tier_label": "Listed only", "first_seen": "2026-08-06T15:22:58.093Z", "resolved_cidrs": [] }, { "id": "ezoicbot", "name": "EzoicBot", "operator": { "name": "Ezoic", "url": "https://www.ezoic.com" }, "description": "Ezoic's crawler family, run by the digital-publisher technology platform of the same name. A desktop and a mobile variant crawl pages to study how sites, search engines and content interact, alongside named subtypes for Core Web Vitals measurement, uptime checks, ads.txt verification, integration checks and page-topic analysis. All variants share the single robots.txt token EzoicBot.", "docs": [ "https://www.ezoic.com/bot" ], "category": "seo", "user_agents": { "patterns": [ "EzoicBot-\\w+", "EzLynx/\\d" ], "instances": [ "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Mobile Safari/537.36 (compatible; EzLynx/0.1; +http://www.ezoic.com/bot.html)", "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Mobile Safari/537.36 (compatible; EzoicBot-IntegrationCheck; +https://www.ezoic.com/bot/)", "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Mobile Safari/537.36 (compatible; EzoicBot-Sitespeed; +http://www.ezoic.com/bot)", "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Mobile Safari/537.36 (compatible; EzoicBot-UptimeOwl; +http://www.ezoic.com/bot)", "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Mobile Safari/537.36 (compatible; EzoicBot-AdsTxt; +http://www.ezoic.com/bot)", "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Mobile Safari/537.36 (compatible; EzoicBot-Nicheiq; +http://www.ezoic.com/bot)" ] }, "behavior": { "respects_robots_txt": true, "robots_token": "EzoicBot" }, "verification": [], "tier": 3, "tier_label": "Listed only", "first_seen": "2026-08-23T13:20:46.873Z", "resolved_cidrs": [] }, { "id": "hubspot-crawler", "name": "HubSpot Crawler", "operator": { "name": "HubSpot", "url": "https://www.hubspot.com" }, "description": "HubSpot's crawler, which fetches customer and external pages to power the SEO recommendations and link analysis in HubSpot's marketing tools. HubSpot publishes its egress ranges tagged by service, including web crawling.", "docs": [ "https://knowledge.hubspot.com/seo/understand-seo-crawling-errors", "https://developers.hubspot.com/docs/api-reference/latest/account/ip-ranges/guide" ], "category": "seo", "user_agents": { "patterns": [ "HubSpot (Crawler|Webcrawler)" ], "instances": [ "Mozilla/5.0 (compatible; HubSpot Crawler; web-crawlers@hubspot.com)" ] }, "behavior": { "respects_robots_txt": true, "robots_token": "HubSpot Crawler" }, "verification": [ { "type": "static_cidrs", "cidrs": [ "3.125.72.185/32", "3.125.100.85/32", "3.125.129.244/32", "3.126.39.79/32", "3.126.163.49/32", "3.126.182.232/32", "3.127.29.218/32", "3.127.31.193/32", "3.127.53.179/32", "3.127.84.21/32", "3.127.178.149/32", "18.156.57.155/32", "18.157.153.160/32", "18.157.207.111/32", "18.157.238.182/32", "18.157.252.152/32", "18.158.38.39/32", "18.158.189.225/32", "18.159.93.15/32", "18.159.128.28/32", "18.159.155.92/32", "18.159.199.77/32", "18.159.231.78/32", "18.159.234.92/32", "18.184.24.22/32", "18.184.86.102/32", "18.185.234.1/32", "18.192.136.149/32", "18.192.141.124/32", "18.192.172.225/32", "18.192.252.214/32", "18.193.15.23/32", "54.174.58.224/27", "216.157.40.64/27", "216.157.41.64/27", "216.157.42.64/27" ] } ], "tier": 2, "tier_label": "Verifiable", "first_seen": "2026-07-18T15:23:43.605Z", "resolved_cidrs": [ "18.156.57.155/32", "18.157.153.160/32", "18.157.207.111/32", "18.157.238.182/32", "18.157.252.152/32", "18.158.189.225/32", "18.158.38.39/32", "18.159.128.28/32", "18.159.155.92/32", "18.159.199.77/32", "18.159.231.78/32", "18.159.234.92/32", "18.159.93.15/32", "18.184.24.22/32", "18.184.86.102/32", "18.185.234.1/32", "18.192.136.149/32", "18.192.141.124/32", "18.192.172.225/32", "18.192.252.214/32", "18.193.15.23/32", "216.157.40.64/27", "216.157.41.64/27", "216.157.42.64/27", "3.125.100.85/32", "3.125.129.244/32", "3.125.72.185/32", "3.126.163.49/32", "3.126.182.232/32", "3.126.39.79/32", "3.127.178.149/32", "3.127.29.218/32", "3.127.31.193/32", "3.127.53.179/32", "3.127.84.21/32", "54.174.58.224/27" ] }, { "id": "integralads-crawler", "name": "IAS Crawler", "operator": { "name": "Integral Ad Science", "url": "https://integralads.com/" }, "description": "Content-rating and ad-verification crawler operated by Integral Ad Science. It visits web pages to assess content quality and brand safety and to support invalid-traffic detection for advertisers.", "docs": [ "https://integralads.com/site-indexing-policy/" ], "category": "seo", "user_agents": { "patterns": [ "ias_crawler" ], "instances": [ "IAS Crawler (ias_crawler; http://integralads.com/site-indexing-policy/)" ] }, "behavior": { "respects_robots_txt": true }, "verification": [], "tier": 3, "tier_label": "Listed only", "first_seen": "2026-07-16T01:14:20.999Z", "resolved_cidrs": [] }, { "id": "linkdex-crawler", "name": "Linkdexbot", "operator": { "name": "Authoritas (Analytics SEO Limited)", "url": "https://www.linkdex.com/" }, "description": "Linkdexbot is the web crawler for Linkdex, an SEO and search-marketing analytics platform now operated by Authoritas. It gathers link and page data used to power the platform's SEO reporting tools.", "docs": [ "https://www.linkdex.com/en-us/about/bots/" ], "category": "seo", "user_agents": { "patterns": [ "linkdexbot" ], "instances": [ "Mozilla/5.0 (compatible; linkdexbot/2.0; +http://www.linkdex.com/about/bots/)" ] }, "behavior": { "respects_robots_txt": true }, "verification": [], "tier": 3, "tier_label": "Listed only", "first_seen": "2026-07-16T00:59:46.504Z", "resolved_cidrs": [] }, { "id": "megaindex-crawler", "name": "MegaIndex Crawler", "operator": { "name": "MegaIndex", "url": "https://www.megaindex.com" }, "description": "MegaIndex is an SEO and web-analytics platform whose crawler indexes links across the web to power backlink analysis, keyword tracking, and site audits for its subscribers.", "docs": [ "https://www.megaindex.com/crawler" ], "category": "seo", "user_agents": { "patterns": [ "MegaIndex\\.ru/" ], "instances": [ "Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +https://www.megaindex.ru/?tab=linkAnalyze)", "Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +http://megaindex.com/crawler)" ] }, "behavior": { "respects_robots_txt": true }, "verification": [], "tier": 3, "tier_label": "Listed only", "first_seen": "2026-07-16T00:59:46.504Z", "resolved_cidrs": [] }, { "id": "meta-externalads", "name": "Meta External Ads", "operator": { "name": "Meta", "url": "https://www.meta.com" }, "description": "Meta's crawler that fetches pages for advertising and other business-related products and services, separate from the AI-training and link-preview crawlers. Verified by ASN lookup (AS32934); Meta publishes no IP feed.", "docs": [ "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/" ], "category": "seo", "user_agents": { "patterns": [ "meta-externalads/\\d" ], "instances": [ "meta-externalads/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)", "meta-externalads/1.1" ] }, "behavior": { "respects_robots_txt": true, "robots_token": "meta-externalads" }, "verification": [ { "type": "asn", "numbers": [ 32934 ] } ], "tier": 2, "tier_label": "Verifiable", "first_seen": "2026-07-26T13:26:40.725Z", "resolved_cidrs": [] }, { "id": "metrics-tools", "name": "MTRobot", "operator": { "name": "Metrics Tools (Andreas Knatz)", "url": "https://metrics-tools.de" }, "description": "Crawler for Metrics Tools, a German SEO analytics service, collecting page data for its visibility and ranking analyses. The operator publishes no IP ranges, so requests cannot be verified beyond the user agent.", "docs": [ "https://metrics-tools.de/robot.html" ], "category": "seo", "user_agents": { "patterns": [ "MTRobot/\\d" ], "instances": [ "MTRobot/0.2 (Metrics Tools Analytics Crawler; https://metrics-tools.de/robot.html; crawler@metrics-tools.de)" ] }, "behavior": { "respects_robots_txt": true }, "verification": [], "tier": 3, "tier_label": "Listed only", "first_seen": "2026-07-26T13:26:40.725Z", "resolved_cidrs": [] }, { "id": "mj12bot", "name": "MJ12bot", "operator": { "name": "Majestic-12", "url": "https://mj12bot.com" }, "description": "The crawler behind Majestic's backlink index. Majestic explicitly states it is a community-based distributed crawler with no fixed IP allocation, so requests cannot be verified by IP, ASN, or reverse DNS.", "docs": [ "https://mj12bot.com/" ], "category": "seo", "user_agents": { "patterns": [ "MJ12bot/v\\d" ], "instances": [ "Mozilla/5.0 (compatible; MJ12bot/v1.4.8; http://mj12bot.com/)" ] }, "behavior": { "respects_robots_txt": true, "robots_token": "MJ12bot" }, "verification": [], "tier": 3, "tier_label": "Listed only", "first_seen": "2026-07-08T19:13:49.571Z", "resolved_cidrs": [] }, { "id": "monsidobot", "name": "Monsidobot", "operator": { "name": "Acquia", "url": "https://www.acquia.com" }, "description": "Acquia Web Governance (formerly Monsido) crawler, which scans the public websites its customers have configured to run accessibility, quality assurance and policy checks. It also issues link-status checks against third-party sites that customers have linked to, preferring HEAD requests for those. Acquia documents a fallback user agent — a plain Chrome string with no identifying token — used only when the primary one fails, so a share of its traffic is identifiable by address rather than by user agent.", "docs": [ "https://docs.acquia.com/web-governance/data-hosting-and-security", "https://docs.acquia.com/web-governance/help/85976-monsidobot-faq" ], "category": "seo", "user_agents": { "patterns": [ "Monsidobot/\\d" ], "instances": [ "Mozilla/5.0 (compatible; Monsidobot/2.2; +http://monsido.com/bot.html; info@monsido.com)" ] }, "behavior": { "respects_robots_txt": false }, "verification": [ { "type": "static_cidrs", "cidrs": [ "35.226.117.128/32", "130.211.65.55/32", "35.189.0.46/32" ] } ], "tier": 2, "tier_label": "Verifiable", "first_seen": "2026-08-23T04:44:25.122Z", "resolved_cidrs": [ "130.211.65.55/32", "35.189.0.46/32", "35.226.117.128/32" ] }, { "id": "nanointeractive-crawler", "name": "Nano Interactive Crawler", "operator": { "name": "Nano Interactive", "url": "https://www.nanointeractive.com" }, "description": "Nano Interactive's contextual-advertising crawler. Its published crawler policy lists four desktop and mobile user agents, all carrying the NanoInteractive/1.0 token, and names the two addresses the crawler requests come from so site owners can allow it explicitly.", "docs": [ "https://www.nanointeractive.com/resources/legal/crawler-policy/" ], "category": "seo", "user_agents": { "patterns": [ "NanoInteractive/\\d" ], "instances": [ "Mozilla/5.0 (iPhone; CPU iPhone OS 16_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.1 Mobile/15E148 Safari/604.1 (compatible; NanoInteractive/1.0; +https://nanointeractive.com/crawler/)", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36 (compatible; NanoInteractive/1.0; +https://nanointeractive.com/crawler/)", "Mozilla/5.0 (Linux; Android 13) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.5414.117 Mobile Safari/537.36 (compatible; NanoInteractive/1.0; +https://nanointeractive.com/crawler/)", "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_2) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.1 Safari/605.1.15 (compatible; NanoInteractive/1.0; +https://nanointeractive.com/crawler/)" ] }, "behavior": { "respects_robots_txt": false }, "verification": [ { "type": "static_cidrs", "cidrs": [ "34.252.254.4/32", "99.81.9.196/32" ] } ], "tier": 2, "tier_label": "Verifiable", "first_seen": "2026-08-23T13:20:46.873Z", "resolved_cidrs": [ "34.252.254.4/32", "99.81.9.196/32" ] }, { "id": "oncrawl", "name": "OnCrawl", "operator": { "name": "OnCrawl", "url": "https://www.oncrawl.com" }, "description": "OnCrawl's SEO crawler, used to analyze a customer's own site structure and content for technical SEO reporting. OnCrawl's help docs describe no fixed IP range; the bot's identity is user-configurable per crawl.", "docs": [ "https://help.oncrawl.com/en/articles/2767653-oncrawl-crawler-how-does-the-oncrawl-bot-find-and-crawl-pages" ], "category": "seo", "user_agents": { "patterns": [ "Oncrawl/\\d" ], "instances": [ "Mozilla/5.0 (compatible; Oncrawl/1.0; +http://www.oncrawl.com/)" ] }, "behavior": { "respects_robots_txt": true, "robots_token": "Oncrawl" }, "verification": [], "tier": 3, "tier_label": "Listed only", "first_seen": "2026-07-08T19:13:49.571Z", "resolved_cidrs": [] }, { "id": "outbrain", "name": "Outbrain crawler", "operator": { "name": "Outbrain", "url": "https://www.outbrain.com" }, "description": "Outbrain's content-recommendation crawler. It fetches advertiser landing pages so Outbrain's system can pull the correct image and headline for a promoted-content unit, and rejects submitted URLs it cannot reach.", "docs": [ "https://www.outbrain.com/help/advertisers/invalid-url/" ], "category": "seo", "user_agents": { "patterns": [ "Mozilla/5\\.0 \\(Java\\) outbrain" ], "instances": [ "Mozilla/5.0 (Java) outbrain" ] }, "behavior": { "respects_robots_txt": false }, "verification": [ { "type": "static_cidrs", "cidrs": [ "192.82.211.0/24", "64.202.112.0/24", "70.42.32.0/24", "66.150.103.0/24", "192.82.210.0/24", "50.31.142.0/24", "64.74.236.0/24", "167.88.57.0/24", "165.254.24.192/27", "66.225.223.0/24", "38.104.143.176/28", "38.133.127.0/24" ] } ], "tier": 2, "tier_label": "Verifiable", "first_seen": "2026-08-06T15:22:58.093Z", "resolved_cidrs": [ "165.254.24.192/27", "167.88.57.0/24", "192.82.210.0/24", "192.82.211.0/24", "38.104.143.176/28", "38.133.127.0/24", "50.31.142.0/24", "64.202.112.0/24", "64.74.236.0/24", "66.150.103.0/24", "66.225.223.0/24", "70.42.32.0/24" ] }, { "id": "panscient", "name": "Panscient Crawler", "operator": { "name": "Panscient Inc.", "url": "https://panscient.com" }, "description": "Panscient's large-scale crawler, which traverses public websites so that Panscient can build structured company and professional data feeds licensed to enterprise customers. The operator documents a full-corpus refresh each quarter, a rate limit of at most one request per second to any single domain, and compliance with the Robot Exclusion Standard. A separate \"pantest\" agent is used for testing. No IP ranges are published.", "docs": [ "https://panscient.com/faq.htm" ], "category": "seo", "user_agents": { "patterns": [ "panscient\\.com", "pantest" ], "instances": [ "panscient.com", "pantest" ] }, "behavior": { "respects_robots_txt": true, "robots_token": "panscient.com" }, "verification": [], "tier": 3, "tier_label": "Listed only", "first_seen": "2026-07-30T16:13:28.285Z", "resolved_cidrs": [] }, { "id": "proximic-crawler", "name": "Proximic (Comscore Crawler)", "operator": { "name": "Comscore, Inc.", "url": "https://www.comscore.com" }, "description": "Proximic is Comscore's web crawler. It downloads the static textual content of pages to perform contextual analysis (content language, rating, and IAB categories) so advertising partners can match campaigns to page content. It identifies itself and honors robots.txt.", "docs": [ "https://www.proximic.com/info/spider.php" ], "category": "seo", "user_agents": { "patterns": [ "proximic" ], "instances": [ "Mozilla/5.0 (compatible; proximic; +https://www.comscore.com/Web-Crawler)", "Mozilla/5.0 (compatible; proximic; +http://www.proximic.com)" ] }, "behavior": { "respects_robots_txt": true, "robots_token": "proximic" }, "verification": [], "tier": 3, "tier_label": "Listed only", "first_seen": "2026-07-16T00:59:46.504Z", "resolved_cidrs": [] }, { "id": "rogerbot", "name": "Rogerbot", "operator": { "name": "Moz", "url": "https://moz.com" }, "description": "Moz's site-audit crawler for Moz Pro Campaigns, distinct from DotBot. Moz's own FAQ states plainly that Rogerbot has no IP range: \"we do not use a static IP address or range of IP addresses.\"", "docs": [ "https://moz.com/help/moz-procedures/crawlers/rogerbot" ], "category": "seo", "user_agents": { "patterns": [ "rogerbot/\\d" ], "instances": [ "rogerbot/1.2 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+phaser-testing-crawler-01@moz.com)" ] }, "behavior": { "respects_robots_txt": true, "robots_token": "rogerbot" }, "verification": [], "tier": 3, "tier_label": "Listed only", "first_seen": "2026-07-08T19:13:49.571Z", "resolved_cidrs": [] }, { "id": "rytebot", "name": "RyteBot", "operator": { "name": "Semrush", "url": "https://www.semrush.com" }, "description": "The crawler behind the Ryte.com tools, which analyse on-page SEO, technical and usability issues. Ryte was absorbed by Semrush, and RyteBot is now documented as a member of the Semrush bot family with its own robots.txt user agent. No IP ranges are published for it.", "docs": [ "https://www.semrush.com/bot/" ], "category": "seo", "user_agents": { "patterns": [ "RyteBot/\\d" ], "instances": [ "RyteBot/1.0.0 (+https://bot.ryte.com/)" ] }, "behavior": { "respects_robots_txt": true, "robots_token": "RyteBot" }, "verification": [], "tier": 3, "tier_label": "Listed only", "first_seen": "2026-08-10T17:35:41.014Z", "resolved_cidrs": [] }, { "id": "scope3-crawler", "name": "Scope3 Crawler", "operator": { "name": "Scope3", "url": "https://scope3.com" }, "description": "Scope3's crawler, which indexes publicly available web content (and paywalled media where the publisher has granted access) to produce content classification and brand-safety assessments for advertising. Scope3 documents adaptive rate limiting of five pages per minute per domain and publishes the single address it crawls from.", "docs": [ "https://docs.scope3.com/docs/scope3-crawler" ], "category": "seo", "user_agents": { "patterns": [ "Scope3/\\d" ], "instances": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36 Scope3/2.0 (scope3.com)" ] }, "behavior": { "respects_robots_txt": true }, "verification": [ { "type": "static_cidrs", "cidrs": [ "34.70.81.55/32" ] } ], "tier": 2, "tier_label": "Verifiable", "first_seen": "2026-08-23T13:20:46.873Z", "resolved_cidrs": [ "34.70.81.55/32" ] }, { "id": "searchatlas", "name": "Search Atlas Bot", "operator": { "name": "Search Atlas", "url": "https://searchatlas.com" }, "description": "The crawler behind Search Atlas's SEO platform, which fetches pages for its Site Auditor and monitoring features. The operator publishes the bot's user agent for allowlisting and states that the crawler does not use static IP addresses, so it can only be identified by its user agent.", "docs": [ "https://searchatlas.com/blog/whitelist-monitoring-on-cloudflare/" ], "category": "seo", "user_agents": { "patterns": [ "Search ?Atlas(\\.com)? ?(Bot|SEO Crawler)" ], "instances": [ "Search Atlas Bot (https://www.searchatlas.com/)", "SearchAtlas.com SEO Crawler" ] }, "behavior": { "respects_robots_txt": false }, "verification": [], "tier": 3, "tier_label": "Listed only", "first_seen": "2026-08-02T14:52:18.385Z", "resolved_cidrs": [] }, { "id": "semrush-siteauditbot", "name": "SiteAuditBot", "operator": { "name": "Semrush", "url": "https://www.semrush.com" }, "description": "Semrush's site-auditing crawler, distinct from SemrushBot: it crawls a domain on demand when a Semrush customer runs the Site Audit tool, looking for SEO and technical issues. Unlike the backlink crawler, which Semrush says cannot be identified by IP, Site Audit is documented as running from a single dedicated subnet.", "docs": [ "https://www.semrush.com/bot/", "https://www.semrush.com/kb/375-why-did-i-get-a-page-is-not-accessible-note-for-some-of-my-pages" ], "category": "seo", "user_agents": { "patterns": [ "SiteAuditBot" ], "instances": [ "SiteAuditBot" ] }, "behavior": { "respects_robots_txt": true, "robots_token": "SiteAuditBot" }, "verification": [ { "type": "static_cidrs", "cidrs": [ "85.208.98.128/25" ] } ], "tier": 2, "tier_label": "Verifiable", "first_seen": "2026-08-10T17:35:41.014Z", "resolved_cidrs": [ "85.208.98.128/25" ] }, { "id": "semrushbot", "name": "SemrushBot", "operator": { "name": "Semrush", "url": "https://www.semrush.com" }, "description": "Semrush's web crawler, feeding the backlink and site-audit data behind the Semrush SEO platform. Semrush's own bot page explicitly states it does not use consecutive IP blocks, so no CIDR list can be sourced.", "docs": [ "https://www.semrush.com/bot/" ], "category": "seo", "user_agents": { "patterns": [ "SemrushBot/\\d" ], "instances": [ "Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html)" ] }, "behavior": { "respects_robots_txt": true, "robots_token": "SemrushBot" }, "verification": [], "tier": 3, "tier_label": "Listed only", "first_seen": "2026-07-08T19:13:49.571Z", "resolved_cidrs": [] }, { "id": "semrushbot-si", "name": "SemrushBot-SI", "operator": { "name": "Semrush", "url": "https://www.semrush.com" }, "description": "The crawler behind Semrush's On Page SEO Checker and related on-page tools, run against a domain when a Semrush customer sets up a campaign for it. It is a separate robots.txt user agent from SemrushBot, and Semrush documents its own addresses to allowlist for it.", "docs": [ "https://www.semrush.com/bot/", "https://www.semrush.com/kb/375-why-did-i-get-a-page-is-not-accessible-note-for-some-of-my-pages" ], "category": "seo", "user_agents": { "patterns": [ "SemrushBot-SI" ], "instances": [ "Mozilla/5.0 (compatible; SemrushBot-SI/0.97; +http://www.semrush.com/bot.html)" ] }, "behavior": { "respects_robots_txt": true, "robots_token": "SemrushBot-SI" }, "verification": [ { "type": "static_cidrs", "cidrs": [ "85.208.98.53", "85.208.98.0/24" ] } ], "tier": 2, "tier_label": "Verifiable", "first_seen": "2026-08-10T17:35:41.014Z", "resolved_cidrs": [ "85.208.98.0/24", "85.208.98.53" ] }, { "id": "seobility", "name": "SeobilityBot", "operator": { "name": "Seobility GmbH", "url": "https://www.seobility.net" }, "description": "Crawler for Seobility's hosted SEO analysis and site-audit tooling, fetching pages of sites its customers analyse. Seobility publishes a machine-readable list of the addresses its bots crawl from.", "docs": [ "https://www.seobility.net/en/bot/" ], "category": "seo", "user_agents": { "patterns": [ "SeobilityBot" ], "instances": [ "SeobilityBot (SEO Tool; https://www.seobility.net/sites/bot.html)" ] }, "behavior": { "respects_robots_txt": true, "robots_token": "Seobility" }, "verification": [ { "type": "cidr_feed", "url": "https://www.seobility.net/bots.json", "format": "prefixes" } ], "tier": 1, "tier_label": "Fully verifiable", "first_seen": "2026-07-18T15:23:43.605Z", "resolved_cidrs": [ "116.202.182.111/32", "116.203.123.238/32", "116.203.125.116/32", "116.203.143.57/32", "116.203.152.117/32", "116.203.176.85/32", "116.203.185.58/32", "116.203.195.17/32", "116.203.44.205/32", "116.203.51.72/32", "116.203.60.150/32", "128.140.104.203/32", "128.140.105.243/32", "128.140.14.171/32", "128.140.38.248/32", "128.140.44.234/32", "128.140.45.196/32", "128.140.64.48/32", "128.140.73.183/32", "128.140.78.247/32", "142.132.167.189/32", "157.90.124.89/32", "157.90.167.94/32", "157.90.173.91/32", "157.90.233.32/32", "157.90.30.158/32", "159.69.147.205/32", "159.69.28.204/32", "159.69.46.182/32", "159.69.6.57/32", "159.69.85.189/32", "159.69.86.158/32", "159.69.9.7/32", "162.55.217.135/32", "162.55.58.96/32", "167.235.132.206/32", "167.235.132.228/32", "167.235.141.22/32", "167.235.151.116/32", "167.235.158.133/32", "167.235.23.143/32", "168.119.171.113/32", "168.119.172.39/32", "178.156.146.210/32", "178.156.164.170/32", "178.156.198.104/32", "178.156.200.243/32", "178.156.202.99/32", "178.156.203.127/32", "178.156.216.32/32", "178.156.218.160/32", "178.156.219.176/32", "178.156.221.57/32", "188.245.107.236/32", "188.245.147.156/32", "188.245.149.2/32", "188.245.151.253/32", "188.245.176.122/32", "188.245.176.6/32", "188.245.208.7/32", "188.245.210.17/32", "188.245.211.238/32", "188.245.211.49/32", "188.245.212.102/32", "188.245.212.107/32", "188.245.212.32/32", "188.245.212.7/32", "188.245.213.57/32", "188.245.214.127/32", "188.245.214.157/32", "188.245.214.95/32", "188.245.33.136/32", "188.245.38.54/32", "188.245.47.30/32", "188.245.64.84/32", "188.245.72.232/32", "188.245.77.46/32", "2a01:4f8:1c0c:4018::1/128", "2a01:4f8:1c0c:49d9::1/128", "2a01:4f8:1c0c:5791::1/128", "2a01:4f8:1c0c:5b9f::1/128", "2a01:4f8:1c0c:6779::1/128", "2a01:4f8:1c0c:71d5::1/128", "2a01:4f8:1c1b:4020::1/128", "2a01:4f8:1c1b:4569::1/128", "2a01:4f8:1c1b:466a::1/128", "2a01:4f8:1c1b:4712::1/128", "2a01:4f8:1c1b:4a41::1/128", "2a01:4f8:1c1b:6271::1/128", "2a01:4f8:1c1b:7bd2::1/128", "2a01:4f8:1c1b:8a93::1/128", "2a01:4f8:1c1b:9365::1/128", "2a01:4f8:1c1b:9c3f::1/128", "2a01:4f8:1c1b:b553::1/128", "2a01:4f8:1c1b:b8bf::1/128", "2a01:4f8:1c1b:d285::1/128", "2a01:4f8:1c1b:d443::1/128", "2a01:4f8:1c1b:d47c::1/128", "2a01:4f8:1c1b:d9ac::1/128", "2a01:4f8:1c1c:1178::1/128", "2a01:4f8:1c1c:184c::1/128", "2a01:4f8:1c1c:495f::1/128", "2a01:4f8:1c1c:6441::1/128", "2a01:4f8:1c1c:6918::1/128", "2a01:4f8:1c1c:7ee7::1/128", "2a01:4f8:1c1c:8a00::1/128", "2a01:4f8:1c1c:8ba3::1/128", "2a01:4f8:1c1c:8f3b::1/128", "2a01:4f8:1c1c:98fb::1/128", "2a01:4f8:1c1c:e75d::1/128", "2a01:4f8:1c1e:4468::1/128", "2a01:4f8:1c1e:494a::1/128", "2a01:4f8:1c1e:6829::1/128", "2a01:4f8:1c1e:6974::1/128", "2a01:4f8:1c1e:7ed8::1/128", "2a01:4f8:1c1e:93fd::1/128", "2a01:4f8:1c1e:a4a3::1/128", "2a01:4f8:1c1e:b9a8::1/128", "2a01:4f8:1c1e:c0eb::1/128", "2a01:4f8:1c1e:d8e9::1/128", "2a01:4f8:1c1e:e5a5::1/128", "2a01:4f8:1c1e:e646::1/128", "2a01:4f8:1c1f:b2df::/64", "2a01:4f8:c0c:1ead::/64", "2a01:4f8:c0c:23b6::/64", "2a01:4f8:c0c:2e24::/64", "2a01:4f8:c0c:4193::1/128", "2a01:4f8:c0c:459e::1/128", "2a01:4f8:c0c:47c3::1/128", "2a01:4f8:c0c:4cb8::/64", "2a01:4f8:c0c:4e7e::1/128", "2a01:4f8:c0c:5e22::/64", "2a01:4f8:c0c:5ff::1/128", "2a01:4f8:c0c:62f4::1/128", "2a01:4f8:c0c:80c1::1/128", "2a01:4f8:c0c:84a7::1/128", "2a01:4f8:c0c:9de3::1/128", "2a01:4f8:c0c:c3e4::1/128", "2a01:4f8:c0c:ede3::1/128", "2a01:4f8:c0c:f4c0::/64", "2a01:4f8:c0c:fd3a::/64", "2a01:4f8:c2c:30d2::1/128", "2a01:4f8:c2c:9351::1/128", "2a01:4f8:c2c:9ff5::1/128", "2a01:4f8:c2c:b136::1/128", "2a01:4f8:c2c:d21::1/128", "2a01:4f8:c2c:dd84::1/128", "2a01:4f8:c2c:e2bc::1/128", "2a01:4f8:c2c:fe8d::1/128", "46.224.117.142/32", "46.224.117.145/32", "46.224.117.162/32", "46.224.117.166/32", "46.224.117.167/32", "46.224.118.207/32", "46.224.120.177/32", "46.224.120.21/32", "46.224.121.221/32", "46.224.124.102/32", "46.224.126.210/32", "46.224.192.231/32", "46.224.208.163/32", "46.224.210.168/32", "46.224.210.184/32", "46.224.210.215/32", "46.224.210.220/32", "46.224.210.88/32", "46.224.211.109/32", "46.224.211.110/32", "46.224.211.122/32", "46.224.211.123/32", "46.224.211.146/32", "46.224.211.147/32", "46.224.211.4/32", "46.224.211.5/32", "46.224.211.53/32", "46.224.212.201/32", "46.224.212.202/32", "46.224.212.208/32", "46.224.212.215/32", "46.224.212.236/32", "46.224.212.242/32", "46.224.212.243/32", "46.224.212.244/32", "46.224.212.245/32", "46.224.212.246/32", "46.224.212.247/32", "46.224.212.25/32", "46.224.212.253/32", "46.224.212.255/32", "46.224.212.27/32", "46.224.212.28/32", "46.224.212.30/32", "46.224.212.73/32", "46.224.212.76/32", "46.224.212.77/32", "46.224.212.78/32", "46.224.212.79/32", "46.224.212.80/32", "46.224.212.81/32", "46.224.213.0/32", "46.224.213.1/32", "46.224.213.153/32", "46.224.213.17/32", "46.224.213.3/32", "46.224.213.4/32", "46.224.213.6/32", "46.224.213.9/32", "46.224.214.127/32", "46.224.214.132/32", "46.224.215.226/32", "46.224.215.73/32", "46.224.216.139/32", "46.224.217.215/32", "46.224.217.49/32", "46.224.218.132/32", "46.224.219.16/32", "46.224.219.53/32", "46.224.221.139/32", "46.224.221.140/32", "46.224.221.145/32", "46.224.222.32/32", "46.224.222.33/32", "46.224.222.34/32", "46.224.222.35/32", "46.224.58.1/32", "46.224.83.115/32", "49.12.193.42/32", "49.12.204.240/32", "49.12.205.98/32", "49.13.164.191/32", "49.13.198.95/32", "5.161.115.9/32", "5.161.126.63/32", "5.161.184.233/32", "5.161.211.219/32", "5.161.213.7/32", "5.161.236.87/32", "5.161.52.46/32", "5.161.65.225/32", "5.161.86.229/32", "5.161.97.73/32", "5.223.48.5/32", "5.223.52.173/32", "5.223.54.133/32", "5.223.58.111/32", "5.223.61.158/32", "5.223.62.119/32", "5.223.62.247/32", "5.223.74.28/32", "5.223.75.235/32", "5.223.79.100/32", "5.75.139.32/32", "5.75.151.175/32", "5.75.171.241/32", "5.75.172.119/32", "5.75.179.74/32", "5.75.183.162/32", "78.46.133.250/32", "78.46.135.69/32", "78.47.109.158/32", "78.47.227.224/32", "78.47.249.97/32", "88.99.171.96/32", "91.107.228.242/32", "91.107.235.117/32", "91.98.21.79/32", "91.98.231.93/32", "91.98.71.210/32", "91.99.161.82/32", "91.99.224.2/32", "94.130.105.241/32", "94.130.111.123/32", "94.130.168.253/32", "94.130.182.75/32", "94.130.186.94/32", "94.130.98.118/32" ] }, { "id": "seokicks-crawler", "name": "SEOkicks", "operator": { "name": "Jobkicks SLU", "url": "https://www.seokicks.de/" }, "description": "SEOkicks operates a web crawler that builds a backlink database powering its SEO tools. The crawler visits sites to collect link data for analysis.", "docs": [ "https://www.seokicks.de/robot.html" ], "category": "seo", "user_agents": { "patterns": [ "SEOkicks" ], "instances": [ "Mozilla/5.0 (compatible; SEOkicks; +https://www.seokicks.de/robot.html)" ] }, "behavior": { "respects_robots_txt": true }, "verification": [], "tier": 3, "tier_label": "Listed only", "first_seen": "2026-07-16T01:14:20.999Z", "resolved_cidrs": [] }, { "id": "serpstatbot", "name": "serpstatbot", "operator": { "name": "Serpstat", "url": "https://serpstat.com" }, "description": "Serpstat's backlink crawler. It continuously crawls the web to add new links and track changes in Serpstat's link database, honouring robots.txt and Crawl-delay directives, and publishes the full list of addresses it crawls from.", "docs": [ "https://serpstatbot.com/" ], "category": "seo", "user_agents": { "patterns": [ "serpstatbot/[\\d.]+" ], "instances": [ "serpstatbot/1.0 (advanced backlink tracking bot; http://serpstatbot.com/; abuse@serpstatbot.com)", "serpstatbot/1.0 (advanced backlink tracking bot; curl/7.58.0; http://serpstatbot.com/; abuse@serpstatbot.com)" ] }, "behavior": { "respects_robots_txt": true, "robots_token": "serpstatbot" }, "verification": [ { "type": "cidr_feed", "url": "https://serpstatbot.com/serpstatbot-ip.txt", "format": "text_lines" } ], "tier": 1, "tier_label": "Fully verifiable", "first_seen": "2026-07-22T13:35:18.584Z", "resolved_cidrs": [ "136.243.144.81/32", "136.243.145.46/32", "136.243.153.17/32", "136.243.155.105/32", "136.243.176.156/32", "136.243.212.110/32", "136.243.212.93/32", "136.243.214.34/32", "138.199.229.125/32", "138.201.194.13/32", "138.201.203.132/32", "138.201.36.87/32", "144.76.13.100/32", "144.76.151.42/32", "144.76.151.45/32", "144.76.61.84/32", "144.76.67.108/32", "144.76.67.110/32", "144.76.67.169/32", "144.76.67.25/32", "144.76.67.250/32", "144.76.68.124/32", "144.76.68.14/32", "144.76.68.17/32", "144.76.68.20/32", "144.76.68.70/32", "144.76.68.76/32", "144.76.68.88/32", "144.76.69.39/32", "144.76.72.24/32", "144.76.72.99/32", "144.76.73.122/32", "144.76.88.54/32", "144.76.98.154/32", "148.251.11.147/32", "148.251.168.205/32", "148.251.241.12/32", "159.69.87.104/32", "159.69.87.105/32", "159.69.87.109/32", "159.69.87.121/32", "159.69.87.155/32", "159.69.87.165/32", "159.69.87.166/32", "159.69.87.192/32", "159.69.87.206/32", "159.69.87.233/32", "159.69.87.26/32", "159.69.87.61/32", "159.69.87.7/32", "159.69.87.78/32", "159.69.87.79/32", "159.69.87.80/32", "159.69.87.90/32", "162.55.54.91/32", "168.119.237.219/32", "176.9.17.6/32", "176.9.18.183/32", "188.245.168.21/32", "188.245.233.162/32", "188.34.166.212/32", "195.201.12.243/32", "195.201.174.92/32", "195.201.235.74/32", "23.88.116.199/32", "49.13.192.12/32", "49.13.90.11/32", "5.9.110.227/32", "5.9.55.228/32", "88.99.149.173/32", "88.99.164.109/32", "88.99.193.224/32", "88.99.215.210/32", "88.99.244.56/32", "88.99.250.124/32", "88.99.95.199/32", "94.130.10.218/32", "94.130.23.168/32", "94.130.237.182/32" ] }, { "id": "sistrix", "name": "SISTRIX Crawler", "operator": { "name": "SISTRIX", "url": "https://www.sistrix.com" }, "description": "The crawler behind the SISTRIX Toolbox, a German SEO visibility platform. SISTRIX documents that every crawler IP resolves via reverse DNS to the \"sistrix.net\" domain rather than publishing a static CIDR list.", "docs": [ "https://www.sistrix.com/tutorials/crawling-errors-in-the-optimizer/" ], "category": "seo", "user_agents": { "patterns": [ "SISTRIX Crawler" ], "instances": [ "Mozilla/5.0 (compatible; SISTRIX Crawler; http://crawler.sistrix.net/)" ] }, "behavior": { "respects_robots_txt": true, "robots_token": "sistrix" }, "verification": [ { "type": "rdns", "masks": [ "*.crawler.sistrix.net" ] } ], "tier": 2, "tier_label": "Verifiable", "first_seen": "2026-07-08T19:13:49.571Z", "resolved_cidrs": [] }, { "id": "siteimprove", "name": "Siteimprove Crawler", "operator": { "name": "Siteimprove", "url": "https://www.siteimprove.com" }, "description": "Siteimprove's content-suite crawler, which fetches pages of sites its customers have configured in their account to run quality-assurance, accessibility, policy and SEO checks. Companion agents (LinkCheck, Image size, Probe) fetch links and resources for the same checks and crawl from the same published address list.", "docs": [ "https://help.siteimprove.com/support/solutions/articles/80001162737-the-siteimprove-crawler-bot-information", "https://help.siteimprove.com/support/solutions/articles/80000448553-what-ip-addresses-and-user-agents-are-used-by-siteimprove-" ], "category": "seo", "user_agents": { "patterns": [ "SiteCheck-sitecrawl by Siteimprove\\.com", "LinkCheck by Siteimprove\\.com", "Image size by Siteimprove\\.com", "Probe by Siteimprove\\.com" ], "instances": [ "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) SiteCheck-sitecrawl by Siteimprove.com", "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) LinkCheck by Siteimprove.com", "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.0) LinkCheck by Siteimprove.com", "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) Image size by Siteimprove.com", "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) Probe by Siteimprove.com" ] }, "behavior": { "respects_robots_txt": true, "robots_token": "SiteimproveBot-Crawler" }, "verification": [ { "type": "static_cidrs", "cidrs": [ "52.55.30.145/32", "52.20.183.91/32", "13.58.165.213/32", "18.116.191.222/32", "18.116.197.208/32", "18.189.206.159/32", "18.190.68.80/32", "18.216.137.252/32", "18.223.191.8/32", "3.13.121.241/32", "3.133.38.181/32", "3.135.49.180/32", "3.136.111.218/32", "3.138.54.100/32", "18.219.35.44/32", "185.229.145.22/32", "3.129.126.175/32", "52.7.141.1/32", "52.4.143.42/32", "52.60.34.56/32", "52.57.25.76/32", "35.157.236.87/32", "35.157.42.7/32", "18.157.140.51/32", "18.159.218.224/32", "18.196.205.2/32", "18.198.120.55/32", "3.124.26.114/32", "3.125.99.135/32", "3.64.159.177/32", "3.66.247.32/32", "3.68.122.244/32", "52.57.167.198/32", "35.158.180.204/32", "18.192.147.131/32", "52.58.146.230/32", "35.156.240.123/32", "185.229.144.10/32", "52.47.166.84/32", "3.10.81.130/32", "52.48.230.149/32", "35.204.24.17/32", "52.64.209.168/32", "52.199.41.160/32" ] } ], "tier": 2, "tier_label": "Verifiable", "first_seen": "2026-07-24T13:19:52.933Z", "resolved_cidrs": [ "13.58.165.213/32", "18.116.191.222/32", "18.116.197.208/32", "18.157.140.51/32", "18.159.218.224/32", "18.189.206.159/32", "18.190.68.80/32", "18.192.147.131/32", "18.196.205.2/32", "18.198.120.55/32", "18.216.137.252/32", "18.219.35.44/32", "18.223.191.8/32", "185.229.144.10/32", "185.229.145.22/32", "3.10.81.130/32", "3.124.26.114/32", "3.125.99.135/32", "3.129.126.175/32", "3.13.121.241/32", "3.133.38.181/32", "3.135.49.180/32", "3.136.111.218/32", "3.138.54.100/32", "3.64.159.177/32", "3.66.247.32/32", "3.68.122.244/32", "35.156.240.123/32", "35.157.236.87/32", "35.157.42.7/32", "35.158.180.204/32", "35.204.24.17/32", "52.199.41.160/32", "52.20.183.91/32", "52.4.143.42/32", "52.47.166.84/32", "52.48.230.149/32", "52.55.30.145/32", "52.57.167.198/32", "52.57.25.76/32", "52.58.146.230/32", "52.60.34.56/32", "52.64.209.168/32", "52.7.141.1/32" ] }, { "id": "t3versions", "name": "t3versionsBot", "operator": { "name": "Torben Hansen (t3versions)", "url": "https://www.t3versions.com" }, "description": "Private-project crawler that makes single GET requests to sites and looks for TYPO3 fingerprints, collecting statistics on the worldwide usage and development of the open-source TYPO3 CMS. No IP ranges are published, and the operator documents no robots.txt support (exclusion is by email request).", "docs": [ "https://www.t3versions.com/bot" ], "category": "seo", "user_agents": { "patterns": [ "t3versionsBot/\\d" ], "instances": [ "Mozilla/5.0 (compatible; t3versionsBot/1.2; +https://www.t3versions.com/bot)" ] }, "behavior": { "respects_robots_txt": false }, "verification": [], "tier": 3, "tier_label": "Listed only", "first_seen": "2026-07-26T13:26:40.725Z", "resolved_cidrs": [] }, { "id": "ttd-content", "name": "TTD-Content", "operator": { "name": "The Trade Desk", "url": "https://www.thetradedesk.com" }, "description": "The Trade Desk's content scraper. When a page sends an ad request to The Trade Desk, this crawler scans the page to determine the context in which the ads were displayed, and caches that contextual data for ad serving. The operator publishes a plain-text list of the addresses it crawls from at ttd-content.adsrvr.org/ips; that list currently holds 2,640 individual addresses, which is beyond this directory's per-feed range cap, so it is not recorded as a machine-readable recipe here.", "docs": [ "https://www.thetradedesk.com/legal/ttd-content" ], "category": "seo", "user_agents": { "patterns": [ "TTD-Content" ], "instances": [ "Mozilla/5.0 (compatible; TTD-Content; +https://www.thetradedesk.com/general/ttd-content)" ] }, "behavior": { "respects_robots_txt": true, "robots_token": "TTD-Content" }, "verification": [], "tier": 3, "tier_label": "Listed only", "first_seen": "2026-08-23T13:20:46.873Z", "resolved_cidrs": [] }, { "id": "velen", "name": "VelenPublicWebCrawler", "operator": { "name": "Hunter", "url": "https://hunter.io" }, "description": "Hunter's public web crawler, written in Go. It analyses millions of publicly accessible pages every month to build the business datasets and machine learning models behind Hunter's products, and never fetches anything behind a login. The operator documents a deliberate rate limit of one page at a time and one page every two seconds per site.", "docs": [ "https://hunter.io/robot", "https://velen.io/" ], "category": "seo", "user_agents": { "patterns": [ "VelenPublicWebCrawler" ], "instances": [ "Mozilla/5.0 (compatible; VelenPublicWebCrawler/1.0; +https://velen.io)", "VelenPublicWebCrawler (velen.io)" ] }, "behavior": { "respects_robots_txt": true, "robots_token": "VelenPublicWebCrawler" }, "verification": [], "tier": 3, "tier_label": "Listed only", "first_seen": "2026-08-02T14:52:18.385Z", "resolved_cidrs": [] }, { "id": "xovibot-crawler", "name": "XoviBot", "operator": { "name": "Xovi GmbH", "url": "https://www.xovi.net" }, "description": "XoviBot is the web crawler for XOVI, an SEO and online-marketing analytics suite. It crawls sites to gather backlink and ranking data for the platform's SEO tools.", "docs": [ "https://www.xovibot.net/" ], "category": "seo", "user_agents": { "patterns": [ "XoviBot" ], "instances": [ "Mozilla/5.0 (compatible; XoviBot/2.0; +http://www.xovibot.net/)" ] }, "behavior": { "respects_robots_txt": true, "robots_token": "XoviBot" }, "verification": [], "tier": 3, "tier_label": "Listed only", "first_seen": "2026-07-16T00:59:46.504Z", "resolved_cidrs": [] }, { "id": "zoominfobot", "name": "Zoominfobot", "operator": { "name": "ZoomInfo Technologies", "url": "https://www.zoominfo.com" }, "description": "ZoomInfo's indexing robot, which scans corporate websites, press releases, news services and SEC filings to build ZoomInfo's search index of businesses and business professionals. The operator documents that it obeys robots.txt, spaces out requests on larger sites and never opens more than one connection to a site at a time. No IP ranges are published.", "docs": [ "https://www.zoominfo.com/legal/zoominfobot" ], "category": "seo", "user_agents": { "patterns": [ "ZoominfoBot" ], "instances": [ "ZoominfoBot (zoominfobot at zoominfo dot com)" ] }, "behavior": { "respects_robots_txt": true, "robots_token": "Zoominfobot" }, "verification": [], "tier": 3, "tier_label": "Listed only", "first_seen": "2026-08-10T17:35:41.014Z", "resolved_cidrs": [] } ] }