{
  "project": "Verified Bots Directory",
  "generated_at": "2026-08-24T04:33:40.448Z",
  "count": 59,
  "bots": [
    {
      "id": "addsearch",
      "name": "AddSearchBot",
      "operator": {
        "name": "AddSearch",
        "url": "https://www.addsearch.com"
      },
      "description": "Crawler for AddSearch's hosted site-search service. It indexes the pages of customer sites so AddSearch can serve search results for them, obeys robots.txt rules written for the AddSearchBot token, and AddSearch publishes the fixed addresses it crawls from for allowlisting.",
      "docs": [
        "https://www.addsearch.com/bot/",
        "https://www.addsearch.com/docs/indexing/excluding-content/"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "AddSearchBot"
        ],
        "instances": [
          "Mozilla/5.0 (compatible; AddSearchBot/1.0; +http://www.addsearch.com/bot/)",
          "Mozilla/5.0 (compatible; AddSearchBot/0.9; +http://www.addsearch.com/bot; info@addsearch.com)"
        ]
      },
      "behavior": {
        "respects_robots_txt": true,
        "robots_token": "AddSearchBot"
      },
      "verification": [
        {
          "type": "static_cidrs",
          "cidrs": [
            "18.211.152.50/32",
            "34.234.198.218/32",
            "34.237.39.240/32",
            "34.237.132.33/32",
            "50.19.247.197/32",
            "52.21.18.234/32",
            "52.22.51.159/32",
            "52.45.51.99/32",
            "52.205.33.118/32"
          ]
        }
      ],
      "tier": 2,
      "tier_label": "Verifiable",
      "first_seen": "2026-07-22T13:35:18.584Z",
      "resolved_cidrs": [
        "18.211.152.50/32",
        "34.234.198.218/32",
        "34.237.132.33/32",
        "34.237.39.240/32",
        "50.19.247.197/32",
        "52.205.33.118/32",
        "52.21.18.234/32",
        "52.22.51.159/32",
        "52.45.51.99/32"
      ]
    },
    {
      "id": "adidxbot",
      "name": "AdIdxBot",
      "operator": {
        "name": "Microsoft",
        "url": "https://www.microsoft.com"
      },
      "description": "Microsoft's crawler for Bing Ads / Microsoft Advertising. It crawls ads and follows through to the advertised landing pages for quality control, with both desktop and mobile variants.",
      "docs": [
        "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0",
        "https://www.bing.com/webmasters/help/verify-bingbot-2195837f"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "adidxbot/\\d"
        ],
        "instances": [
          "Mozilla/5.0 (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)",
          "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)"
        ]
      },
      "behavior": {
        "respects_robots_txt": true
      },
      "verification": [
        {
          "type": "rdns",
          "masks": [
            "*.search.msn.com"
          ]
        }
      ],
      "tier": 2,
      "tier_label": "Verifiable",
      "first_seen": "2026-07-09T03:01:47.690Z",
      "resolved_cidrs": []
    },
    {
      "id": "algolia-crawler",
      "name": "Algolia Crawler",
      "operator": {
        "name": "Algolia",
        "url": "https://www.algolia.com"
      },
      "description": "Algolia's Crawler, which visits a customer's pages, extracts search-relevant content, and pushes it to Algolia search indices. Algolia documents a fixed User-Agent token and a single static egress IP address for allowlisting.",
      "docs": [
        "https://www.algolia.com/doc/tools/crawler/getting-started/overview/",
        "https://support.algolia.com/hc/en-us/articles/17223760387857-What-is-the-user-agent-of-the-Crawler",
        "https://support.algolia.com/hc/en-us/articles/17223698862993-Which-IP-address-of-the-Crawler-can-I-add-to-my-allowlist"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "Algolia Crawler/[\\d.]+"
        ],
        "instances": [
          "Algolia Crawler/1.0.0"
        ]
      },
      "behavior": {
        "respects_robots_txt": true,
        "robots_token": "Algolia Crawler"
      },
      "verification": [
        {
          "type": "static_cidrs",
          "cidrs": [
            "34.66.202.43/32"
          ]
        }
      ],
      "tier": 2,
      "tier_label": "Verifiable",
      "first_seen": "2026-07-13T17:31:56.719Z",
      "resolved_cidrs": [
        "34.66.202.43/32"
      ]
    },
    {
      "id": "amazon-adbot",
      "name": "Amazon AdBot",
      "operator": {
        "name": "Amazon",
        "url": "https://www.amazon.com"
      },
      "description": "Amazon's advertising crawler. It scans web pages that request ads from Amazon's advertising systems, collecting page content for Amazon's classification systems to maintain brand safety and improve ad relevance.",
      "docs": [
        "https://adbot.amazon.com/"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "AmazonAdBot/\\d"
        ],
        "instances": [
          "Mozilla/5.0 (compatible; AmazonAdBot/1.0; +https://adbot.amazon.com)"
        ]
      },
      "behavior": {
        "respects_robots_txt": true,
        "robots_token": "AmazonAdBot"
      },
      "verification": [
        {
          "type": "rdns",
          "masks": [
            "*.amazonadbot.com"
          ]
        }
      ],
      "tier": 2,
      "tier_label": "Verifiable",
      "first_seen": "2026-07-09T03:01:47.690Z",
      "resolved_cidrs": []
    },
    {
      "id": "amazon-product-discovery",
      "name": "AmazonProductDiscoverybot",
      "operator": {
        "name": "Amazon",
        "url": "https://www.amazon.com"
      },
      "description": "Amazon's product crawler, which collects publicly available product details from Amazon selling partner, brand and retailer websites to improve the accuracy and completeness of product information in Amazon's store. Amazon documents that it honours robots.txt user-agent and disallow directives, with changes taking up to 24 hours to apply, and that it does not support crawl-delay, nofollow or noindex.",
      "docs": [
        "https://vendorcentral.amazon.com/support/amazonproductbot"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "AmazonProductDiscovery/\\d"
        ],
        "instances": [
          "Mozilla/5.0 (X11; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/38.0 (compatible; AmazonProductDiscovery/1.0; https://vendorcentral.amazon.com/support/amazonproductbot)"
        ]
      },
      "behavior": {
        "respects_robots_txt": true,
        "robots_token": "AmazonProductDiscoverybot"
      },
      "verification": [],
      "tier": 3,
      "tier_label": "Listed only",
      "first_seen": "2026-08-23T13:20:46.873Z",
      "resolved_cidrs": []
    },
    {
      "id": "amzn-searchbot",
      "name": "Amzn-SearchBot",
      "operator": {
        "name": "Amazon",
        "url": "https://www.amazon.com"
      },
      "description": "Amazon's crawler used to improve search experiences in Amazon products and services; Amazon states it does not crawl content for generative AI model training. Amazon publishes a human-readable IP list but no machine-parseable feed in a format this project's schema supports.",
      "docs": [
        "https://developer.amazon.com/amazonbot"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "Amzn-SearchBot/\\d"
        ],
        "instances": [
          "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amzn-SearchBot/0.1) Chrome/W.X.Y.Z Safari/537.36"
        ]
      },
      "behavior": {
        "respects_robots_txt": true,
        "robots_token": "Amzn-SearchBot"
      },
      "verification": [],
      "tier": 3,
      "tier_label": "Listed only",
      "first_seen": "2026-07-26T13:26:40.725Z",
      "resolved_cidrs": []
    },
    {
      "id": "anomura",
      "name": "Anomura",
      "operator": {
        "name": "Direqt",
        "url": "https://www.direqt.ai"
      },
      "description": "Direqt's search crawler, which discovers links and metadata to surface in Direqt's search features. The operator states it is not used to crawl content for model training, documents the robots.txt token Anomura, and publishes the two addresses the crawler requests come from.",
      "docs": [
        "https://docs.direqt-search.com/direqt-bots/direqt-crawlers-and-user-agents"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "Anomura/\\d"
        ],
        "instances": [
          "Anomura/1.2 (+https://www.direqt.ai)",
          "Anomura/1.2 (+https://www.direqt-search.com)"
        ]
      },
      "behavior": {
        "respects_robots_txt": true,
        "robots_token": "Anomura"
      },
      "verification": [
        {
          "type": "static_cidrs",
          "cidrs": [
            "34.56.242.161/32",
            "35.224.164.246/32"
          ]
        }
      ],
      "tier": 2,
      "tier_label": "Verifiable",
      "first_seen": "2026-08-23T13:20:46.873Z",
      "resolved_cidrs": [
        "34.56.242.161/32",
        "35.224.164.246/32"
      ]
    },
    {
      "id": "applebot",
      "name": "Applebot",
      "operator": {
        "name": "Apple",
        "url": "https://www.apple.com"
      },
      "description": "Apple's web crawler, used to index content for Siri, Spotlight Suggestions, and Safari's search features.",
      "docs": [
        "https://support.apple.com/en-us/119829"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "Applebot/\\d"
        ],
        "instances": [
          "Mozilla/5.0 (compatible; Applebot/0.3; +http://www.apple.com/go/applebot)",
          "Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B410 Safari/600.1.4 (Applebot/0.1; +http://www.apple.com/go/applebot)"
        ]
      },
      "behavior": {
        "respects_robots_txt": true,
        "robots_token": "Applebot"
      },
      "verification": [
        {
          "type": "cidr_feed",
          "url": "https://search.developer.apple.com/applebot.json",
          "format": "prefixes"
        },
        {
          "type": "rdns",
          "masks": [
            "*.applebot.apple.com"
          ]
        }
      ],
      "tier": 1,
      "tier_label": "Fully verifiable",
      "first_seen": "2026-07-08T19:13:49.571Z",
      "resolved_cidrs": [
        "17.22.237.0/24",
        "17.22.245.0/24",
        "17.22.253.0/24",
        "17.241.193.160/27",
        "17.241.200.160/27",
        "17.241.208.160/27",
        "17.241.219.0/24",
        "17.241.227.0/24",
        "17.241.75.0/24",
        "17.246.15.0/24",
        "17.246.19.0/24",
        "17.246.23.0/24"
      ]
    },
    {
      "id": "baiduspider",
      "name": "Baiduspider",
      "operator": {
        "name": "Baidu",
        "url": "https://www.baidu.com"
      },
      "description": "Baidu's primary web crawler, fetching pages for Baidu Search indexing.",
      "docs": [
        "https://help.baidu.com/question?prod_en=master&class=Baiduspider"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "Baiduspider/\\d"
        ],
        "instances": [
          "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)",
          "Mozilla/5.0 (compatible; Baiduspider-render/2.0; +http://www.baidu.com/search/spider.html)"
        ]
      },
      "behavior": {
        "respects_robots_txt": true,
        "robots_token": "Baiduspider"
      },
      "verification": [
        {
          "type": "rdns",
          "masks": [
            "*.baidu.com",
            "*.baidu.jp"
          ]
        }
      ],
      "tier": 2,
      "tier_label": "Verifiable",
      "first_seen": "2026-07-08T19:13:49.571Z",
      "resolved_cidrs": []
    },
    {
      "id": "bing-video-preview",
      "name": "BingVideoPreview",
      "operator": {
        "name": "Microsoft",
        "url": "https://www.microsoft.com"
      },
      "description": "Microsoft crawler, listed among Bing's crawlers, that fetches pages to generate video previews shown in Bing. It runs desktop and mobile variants and is verifiable through the same reverse-DNS check as Bing's other crawlers.",
      "docs": [
        "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0",
        "https://www.bing.com/webmasters/help/how-to-verify-bingbot-3905dc26"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "BingVideoPreview/\\d"
        ],
        "instances": [
          "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; BingVideoPreview/1.0; +https://aka.ms/microsoftbots) Chrome/W.X.Y.Z Safari/537.36",
          "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36  (compatible; BingVideoPreview/1.0; +https://aka.ms/microsoftbots)"
        ]
      },
      "behavior": {
        "respects_robots_txt": true
      },
      "verification": [
        {
          "type": "rdns",
          "masks": [
            "*.search.msn.com"
          ]
        }
      ],
      "tier": 2,
      "tier_label": "Verifiable",
      "first_seen": "2026-08-04T20:03:13.870Z",
      "resolved_cidrs": []
    },
    {
      "id": "bingbot",
      "name": "Bingbot",
      "operator": {
        "name": "Microsoft",
        "url": "https://www.microsoft.com"
      },
      "description": "Microsoft's primary web crawler, fetching pages for Bing Search indexing.",
      "docs": [
        "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "bingbot/\\d"
        ],
        "instances": [
          "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Safari/537.36",
          "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)"
        ]
      },
      "behavior": {
        "respects_robots_txt": true,
        "robots_token": "bingbot"
      },
      "verification": [
        {
          "type": "cidr_feed",
          "url": "https://www.bing.com/toolbox/bingbot.json",
          "format": "prefixes"
        },
        {
          "type": "rdns",
          "masks": [
            "*.search.msn.com"
          ]
        }
      ],
      "tier": 1,
      "tier_label": "Fully verifiable",
      "first_seen": "2026-07-08T19:13:49.571Z",
      "resolved_cidrs": [
        "13.66.139.0/24",
        "13.66.144.0/24",
        "13.67.10.16/28",
        "13.69.66.240/28",
        "13.71.172.224/28",
        "139.217.52.0/28",
        "157.55.39.0/24",
        "191.233.204.224/28",
        "199.30.24.0/23",
        "20.125.163.80/28",
        "20.15.133.160/27",
        "20.36.108.32/28",
        "20.43.120.16/28",
        "20.74.197.0/28",
        "20.79.107.240/28",
        "207.46.13.0/24",
        "40.77.139.0/25",
        "40.77.167.0/24",
        "40.77.177.0/24",
        "40.77.178.0/23",
        "40.77.188.0/22",
        "40.77.202.0/24",
        "40.79.131.208/28",
        "40.79.186.176/28",
        "51.105.67.0/28",
        "52.167.144.0/24",
        "52.231.148.0/28",
        "65.55.210.0/24"
      ]
    },
    {
      "id": "bingpreview",
      "name": "BingPreview",
      "operator": {
        "name": "Microsoft",
        "url": "https://www.microsoft.com"
      },
      "description": "Microsoft fetcher used to generate page snapshots for previews in Bing search results. Runs both desktop and mobile variants.",
      "docs": [
        "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0",
        "https://www.bing.com/webmasters/help/verify-bingbot-2195837f"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "BingPreview/\\d"
        ],
        "instances": [
          "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534+ (KHTML, like Gecko) BingPreview/1.0b",
          "Mozilla/5.0 (Windows Phone 8.1; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 530) like Gecko BingPreview/1.0b"
        ]
      },
      "behavior": {
        "respects_robots_txt": true
      },
      "verification": [
        {
          "type": "rdns",
          "masks": [
            "*.search.msn.com"
          ]
        }
      ],
      "tier": 2,
      "tier_label": "Verifiable",
      "first_seen": "2026-07-09T03:01:47.690Z",
      "resolved_cidrs": []
    },
    {
      "id": "bublupbot",
      "name": "Bublup Bot",
      "operator": {
        "name": "Bublup",
        "url": "https://www.bublup.com"
      },
      "description": "Bublup's content-discovery bot. It fetches pages to build the database behind Bublup's suggestion engine, reading page title, description and related images. Bublup states its crawling IPs are not fixed and documents reverse-DNS verification against the bublup.com domain instead.",
      "docs": [
        "https://www.bublup.com/bublup-bot"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "BublupBot"
        ],
        "instances": [
          "BublupBot (+https://www.bublup.com/bublup-bot.html)",
          "Mozilla/5.0 (compatible; BublupBot; https://www.bublup.com/bublup-bot.html)"
        ]
      },
      "behavior": {
        "respects_robots_txt": false
      },
      "verification": [
        {
          "type": "rdns",
          "masks": [
            "*.bublup.com"
          ]
        }
      ],
      "tier": 2,
      "tier_label": "Verifiable",
      "first_seen": "2026-07-28T19:51:24.263Z",
      "resolved_cidrs": []
    },
    {
      "id": "channel3bot",
      "name": "Channel3Bot",
      "operator": {
        "name": "Channel3",
        "url": "https://trychannel3.com"
      },
      "description": "Channel3's product crawler. It visits publicly accessible product detail pages and collects images, titles, descriptions, prices, availability and variants for Channel3's product catalogue, which AI apps and agents query to route shoppers back to the original site. The operator tells site owners not to hard-code its addresses and to verify it by forward-confirmed reverse DNS instead.",
      "docs": [
        "https://trychannel3.com/channel3bot"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "Channel3Bot/\\d",
          "Channel3/\\d"
        ],
        "instances": [
          "Mozilla/5.0 (compatible; Channel3Bot/1.0; +https://trychannel3.com/channel3bot)",
          "Channel3/1.0 (+https://trychannel3.com/channel3bot)"
        ]
      },
      "behavior": {
        "respects_robots_txt": true
      },
      "verification": [
        {
          "type": "rdns",
          "masks": [
            "*.bot.trychannel3.com"
          ]
        }
      ],
      "tier": 2,
      "tier_label": "Verifiable",
      "first_seen": "2026-08-23T13:20:46.873Z",
      "resolved_cidrs": []
    },
    {
      "id": "coccocbot",
      "name": "coccocbot",
      "operator": {
        "name": "Cốc Cốc",
        "url": "https://coccoc.com"
      },
      "description": "Cốc Cốc's web crawler, fetching pages for the Vietnamese Cốc Cốc search engine. Separate web and image sub-bots share the same verification domain.",
      "docs": [
        "https://coccoc.com/search/console/en"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "coccocbot-web/\\d",
          "coccocbot-image/\\d"
        ],
        "instances": [
          "Mozilla/5.0 (compatible; coccocbot-web/1.0; +http://help.coccoc.com/searchengine)",
          "Mozilla/5.0 (compatible; coccocbot-image/1.0; +http://help.coccoc.com/searchengine)"
        ]
      },
      "behavior": {
        "respects_robots_txt": true,
        "robots_token": "coccocbot"
      },
      "verification": [
        {
          "type": "rdns",
          "masks": [
            "*.coccoc.com"
          ]
        }
      ],
      "tier": 2,
      "tier_label": "Verifiable",
      "first_seen": "2026-07-08T19:13:49.571Z",
      "resolved_cidrs": []
    },
    {
      "id": "deepnoc",
      "name": "deepnoc",
      "operator": {
        "name": "deepnoc GmbH",
        "url": "https://deepnoc.com"
      },
      "description": "Research crawler operated by deepnoc GmbH that parses and stores public web page content so that new search engines can query it without running their own crawling infrastructure. No IP ranges are published.",
      "docs": [
        "https://deepnoc.com/bot"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "deepnoc"
        ],
        "instances": [
          "deepnoc - https://deepnoc.com/bot"
        ]
      },
      "behavior": {
        "respects_robots_txt": true,
        "robots_token": "deepnoc"
      },
      "verification": [],
      "tier": 3,
      "tier_label": "Listed only",
      "first_seen": "2026-07-26T13:26:40.725Z",
      "resolved_cidrs": []
    },
    {
      "id": "duckduckbot",
      "name": "DuckDuckBot",
      "operator": {
        "name": "DuckDuckGo",
        "url": "https://duckduckgo.com"
      },
      "description": "DuckDuckGo's web crawler, used to improve DuckDuckGo's search results.",
      "docs": [
        "https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot/"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "DuckDuckBot/\\d"
        ],
        "instances": [
          "DuckDuckBot/1.1; (+http://duckduckgo.com/duckduckbot.html)",
          "Mozilla/5.0 (compatible; DuckDuckBot-Https/1.1; https://duckduckgo.com/duckduckbot)"
        ]
      },
      "behavior": {
        "respects_robots_txt": true,
        "robots_token": "DuckDuckBot"
      },
      "verification": [
        {
          "type": "cidr_feed",
          "url": "https://duckduckgo.com/duckduckbot.json",
          "format": "prefixes"
        }
      ],
      "tier": 1,
      "tier_label": "Fully verifiable",
      "first_seen": "2026-07-08T19:13:49.571Z",
      "resolved_cidrs": [
        "104.43.54.127/32",
        "104.43.55.116/32",
        "104.43.55.117/32",
        "104.43.55.166/32",
        "104.43.55.167/32",
        "108.141.83.74/32",
        "128.203.132.152/32",
        "128.203.181.79/32",
        "128.203.214.241/32",
        "128.203.236.110/32",
        "128.203.82.154/32",
        "13.86.35.212/32",
        "13.89.106.77/32",
        "13.90.178.139/32",
        "130.107.228.224/32",
        "131.145.39.118/32",
        "131.189.183.241/32",
        "132.164.209.198/32",
        "132.196.106.146/32",
        "132.220.1.71/32",
        "134.33.207.189/32",
        "135.233.63.237/32",
        "135.234.221.112/32",
        "135.235.183.95/32",
        "172.168.115.250/32",
        "172.168.12.93/32",
        "172.168.133.76/32",
        "172.168.137.102/32",
        "172.168.152.88/32",
        "172.168.173.230/32",
        "172.168.190.186/32",
        "172.168.227.120/32",
        "172.168.254.119/32",
        "172.168.43.43/32",
        "172.168.51.216/32",
        "172.168.53.53/32",
        "172.168.81.217/32",
        "172.168.94.160/32",
        "172.169.164.109/32",
        "172.169.17.165/32",
        "172.169.177.131/32",
        "172.169.181.237/32",
        "172.169.28.184/32",
        "172.169.36.43/32",
        "172.169.53.208/32",
        "172.169.60.134/32",
        "172.169.7.33/32",
        "172.169.93.148/32",
        "172.171.37.221/32",
        "172.179.105.87/32",
        "172.179.128.118/32",
        "172.189.40.18/32",
        "172.193.184.42/32",
        "172.193.197.146/32",
        "172.193.245.229/32",
        "172.194.141.49/32",
        "172.199.55.212/32",
        "172.210.111.68/32",
        "191.233.3.197/32",
        "191.233.3.202/32",
        "191.234.216.178/32",
        "191.234.216.4/32",
        "191.235.201.214/32",
        "191.235.202.38/32",
        "191.235.202.48/32",
        "20.100.136.36/32",
        "20.100.140.155/32",
        "20.101.17.173/32",
        "20.102.34.205/32",
        "20.109.129.162/32",
        "20.109.130.190/32",
        "20.109.241.18/32",
        "20.112.58.44/32",
        "20.113.14.159/32",
        "20.113.3.121/32",
        "20.115.181.187/32",
        "20.118.11.251/32",
        "20.119.127.146/32",
        "20.12.141.99/32",
        "20.12.249.123/32",
        "20.12.252.21/32",
        "20.121.75.32/32",
        "20.13.44.19/32",
        "20.15.142.178/32",
        "20.166.171.150/32",
        "20.170.75.54/32",
        "20.175.232.228/32",
        "20.185.79.15/32",
        "20.185.79.47/32",
        "20.190.21.172/32",
        "20.191.44.119/32",
        "20.191.44.16/32",
        "20.191.44.22/32",
        "20.191.44.234/32",
        "20.191.45.212/32",
        "20.193.12.126/32",
        "20.193.24.10/32",
        "20.193.24.251/32",
        "20.193.25.197/32",
        "20.193.27.215/32",
        "20.193.45.113/32",
        "20.195.108.47/32",
        "20.195.58.189/32",
        "20.197.209.11/32",
        "20.197.209.27/32",
        "20.198.132.126/32",
        "20.198.152.208/32",
        "20.201.15.208/32",
        "20.204.228.200/32",
        "20.204.240.172/32",
        "20.204.241.148/32",
        "20.204.242.101/32",
        "20.204.242.19/32",
        "20.204.243.55/32",
        "20.204.246.254/32",
        "20.204.246.81/32",
        "20.207.107.181/32",
        "20.207.117.66/32",
        "20.207.72.11/32",
        "20.207.72.110/32",
        "20.207.72.113/32",
        "20.207.72.21/32",
        "20.207.97.190/32",
        "20.207.99.197/32",
        "20.212.139.34/32",
        "20.212.90.107/32",
        "20.216.200.223/32",
        "20.219.43.246/32",
        "20.219.45.190/32",
        "20.219.45.67/32",
        "20.221.10.115/32",
        "20.221.84.185/32",
        "20.226.133.105/32",
        "20.23.88.231/32",
        "20.231.254.179/32",
        "20.232.249.95/32",
        "20.232.51.46/32",
        "20.236.195.196/32",
        "20.236.198.192/32",
        "20.236.227.113/32",
        "20.236.233.23/32",
        "20.236.247.180/32",
        "20.236.40.93/32",
        "20.237.1.51/32",
        "20.241.135.19/32",
        "20.241.147.151/32",
        "20.241.153.203/32",
        "20.241.164.5/32",
        "20.241.241.100/32",
        "20.242.133.53/32",
        "20.242.135.165/32",
        "20.242.211.139/32",
        "20.242.215.53/32",
        "20.242.237.240/32",
        "20.246.132.168/32",
        "20.246.225.34/32",
        "20.247.235.11/32",
        "20.250.51.113/32",
        "20.252.16.193/32",
        "20.252.41.18/32",
        "20.252.63.162/32",
        "20.252.88.10/32",
        "20.252.9.6/32",
        "20.253.114.51/32",
        "20.253.59.76/32",
        "20.253.96.199/32",
        "20.29.149.44/32",
        "20.29.154.56/32",
        "20.3.1.178/32",
        "20.3.60.212/32",
        "20.40.133.240/32",
        "20.40.147.172/32",
        "20.43.150.85/32",
        "20.43.150.93/32",
        "20.43.152.28/32",
        "20.43.172.120/32",
        "20.44.222.1/32",
        "20.49.129.236/32",
        "20.49.136.28/32",
        "20.50.168.91/32",
        "20.50.48.159/32",
        "20.50.48.192/32",
        "20.50.49.0/32",
        "20.50.49.237/32",
        "20.50.49.25/32",
        "20.50.49.40/32",
        "20.50.49.55/32",
        "20.50.50.118/32",
        "20.50.50.121/32",
        "20.50.50.123/32",
        "20.50.50.130/32",
        "20.50.50.134/32",
        "20.50.50.145/32",
        "20.50.50.146/32",
        "20.50.50.163/32",
        "20.50.50.46/32",
        "20.53.134.160/32",
        "20.53.78.106/32",
        "20.53.78.123/32",
        "20.53.78.138/32",
        "20.53.78.144/32",
        "20.53.78.236/32",
        "20.53.91.2/32",
        "20.53.92.211/32",
        "20.54.224.39/32",
        "20.56.197.58/32",
        "20.56.197.63/32",
        "20.59.0.65/32",
        "20.61.142.192/32",
        "20.61.34.40/32",
        "20.62.224.44/32",
        "20.64.185.45/32",
        "20.69.129.205/32",
        "20.69.131.45/32",
        "20.69.138.149/32",
        "20.69.138.189/32",
        "20.69.144.137/32",
        "20.69.146.171/32",
        "20.69.147.21/32",
        "20.69.171.204/32",
        "20.69.74.250/32",
        "20.71.12.143/32",
        "20.71.69.210/32",
        "20.72.197.110/32",
        "20.72.203.250/32",
        "20.72.218.29/32",
        "20.72.221.33/32",
        "20.72.221.45/32",
        "20.72.225.13/32",
        "20.72.242.209/32",
        "20.72.242.93/32",
        "20.73.132.240/32",
        "20.73.202.147/32",
        "20.75.144.152/32",
        "20.75.254.61/32",
        "20.77.146.108/32",
        "20.79.226.26/32",
        "20.79.238.198/32",
        "20.79.239.66/32",
        "20.8.252.26/32",
        "20.80.114.203/32",
        "20.80.120.41/32",
        "20.80.129.80/32",
        "20.80.135.7/32",
        "20.80.156.88/32",
        "20.81.115.13/32",
        "20.82.24.207/32",
        "20.84.138.61/32",
        "20.84.219.45/32",
        "20.84.251.223/32",
        "20.85.138.75/32",
        "20.85.167.56/32",
        "20.85.187.146/32",
        "20.88.168.65/32",
        "20.9.114.242/32",
        "20.93.28.24/32",
        "20.93.49.96/32",
        "20.99.154.113/32",
        "20.99.255.235/32",
        "4.144.182.50/32",
        "4.149.142.7/32",
        "4.149.159.102/32",
        "4.149.159.71/32",
        "4.149.216.220/32",
        "4.150.142.218/32",
        "4.154.193.167/32",
        "4.156.136.79/32",
        "4.156.154.107/32",
        "4.156.30.123/32",
        "4.157.77.76/32",
        "4.172.49.103/32",
        "4.177.16.196/32",
        "4.182.10.198/32",
        "4.182.131.108/32",
        "4.195.133.120/32",
        "4.201.125.59/32",
        "4.201.141.71/32",
        "4.201.197.203/32",
        "4.201.206.133/32",
        "4.201.220.8/32",
        "4.207.220.92/32",
        "4.209.224.56/32",
        "4.213.46.14/32",
        "4.226.40.135/32",
        "4.228.76.163/32",
        "4.237.155.54/32",
        "4.237.244.80/32",
        "4.242.119.13/32",
        "4.246.10.252/32",
        "4.248.234.26/32",
        "4.249.216.104/32",
        "4.249.223.51/32",
        "4.249.238.11/32",
        "4.254.123.56/32",
        "4.255.35.121/32",
        "4.255.39.80/32",
        "40.114.182.153/32",
        "40.114.182.172/32",
        "40.114.182.45/32",
        "40.114.183.196/32",
        "40.114.183.251/32",
        "40.114.183.88/32",
        "40.119.232.146/32",
        "40.119.232.215/32",
        "40.119.232.218/32",
        "40.119.232.251/32",
        "40.119.232.50/32",
        "40.121.121.54/32",
        "40.127.154.196/32",
        "40.64.105.247/32",
        "40.64.106.11/32",
        "40.64.106.152/32",
        "40.64.67.220/32",
        "40.64.78.45/32",
        "40.64.88.145/32",
        "40.65.78.86/32",
        "40.76.162.17/32",
        "40.76.162.191/32",
        "40.76.162.208/32",
        "40.76.162.247/32",
        "40.76.163.23/32",
        "40.76.163.7/32",
        "40.76.173.151/32",
        "40.81.250.205/32",
        "40.82.218.203/32",
        "40.88.195.62/32",
        "40.88.21.235/32",
        "40.89.243.175/32",
        "48.192.16.86/32",
        "48.192.74.178/32",
        "48.200.33.227/32",
        "48.202.155.200/32",
        "48.202.171.124/32",
        "48.204.32.87/32",
        "48.206.18.152/32",
        "48.217.129.210/32",
        "48.217.212.89/32",
        "48.217.23.236/32",
        "48.223.209.110/32",
        "51.104.144.31/32",
        "51.104.146.225/32",
        "51.104.146.235/32",
        "51.104.160.167/32",
        "51.104.160.177/32",
        "51.104.161.32/32",
        "51.104.162.149/32",
        "51.104.163.250/32",
        "51.104.164.109/32",
        "51.104.164.147/32",
        "51.104.164.189/32",
        "51.104.164.215/32",
        "51.104.166.111/32",
        "51.104.167.104/32",
        "51.104.167.110/32",
        "51.104.167.19/32",
        "51.104.167.52/32",
        "51.104.167.54/32",
        "51.104.167.61/32",
        "51.104.167.71/32",
        "51.104.167.87/32",
        "51.104.167.88/32",
        "51.104.167.95/32",
        "51.104.167.96/32",
        "51.104.180.26/32",
        "51.104.180.47/32",
        "51.104.180.53/32",
        "51.107.40.209/32",
        "51.116.131.221/32",
        "51.120.48.122/32",
        "51.138.90.161/32",
        "51.138.90.206/32",
        "51.138.90.233/32",
        "51.143.114.45/32",
        "51.145.185.16/32",
        "51.8.253.152/32",
        "51.8.31.28/32",
        "51.8.71.117/32",
        "52.142.24.149/32",
        "52.142.26.175/32",
        "52.143.241.111/32",
        "52.143.242.6/32",
        "52.143.243.117/32",
        "52.143.244.81/32",
        "52.143.247.235/32",
        "52.143.95.162/32",
        "52.143.95.204/32",
        "52.146.57.51/32",
        "52.146.58.236/32",
        "52.146.59.12/32",
        "52.146.59.154/32",
        "52.146.59.156/32",
        "52.146.63.80/32",
        "52.148.161.87/32",
        "52.148.165.38/32",
        "52.149.25.43/32",
        "52.149.28.18/32",
        "52.149.28.83/32",
        "52.149.30.45/32",
        "52.149.56.151/32",
        "52.149.58.139/32",
        "52.149.58.173/32",
        "52.149.58.27/32",
        "52.149.58.69/32",
        "52.149.60.38/32",
        "52.149.61.51/32",
        "52.154.169.200/32",
        "52.154.169.50/32",
        "52.154.170.113/32",
        "52.154.170.117/32",
        "52.154.170.122/32",
        "52.154.170.209/32",
        "52.154.170.229/32",
        "52.154.170.243/32",
        "52.154.170.26/32",
        "52.154.170.28/32",
        "52.154.170.88/32",
        "52.154.170.96/32",
        "52.154.171.0/32",
        "52.154.171.150/32",
        "52.154.171.196/32",
        "52.154.171.205/32",
        "52.154.171.235/32",
        "52.154.171.250/32",
        "52.154.171.44/32",
        "52.154.171.70/32",
        "52.154.171.87/32",
        "52.154.172.2/32",
        "52.154.204.93/32",
        "52.154.60.82/32",
        "52.185.93.208/32",
        "52.186.37.211/32",
        "52.188.89.106/32",
        "52.189.70.103/32",
        "52.190.37.160/32",
        "52.191.222.158/32",
        "52.191.82.242/32",
        "52.224.147.21/32",
        "52.224.16.221/32",
        "52.224.16.229/32",
        "52.224.19.152/32",
        "52.224.191.245/32",
        "52.224.20.174/32",
        "52.224.20.181/32",
        "52.224.20.186/32",
        "52.224.20.190/32",
        "52.224.20.193/32",
        "52.224.20.203/32",
        "52.224.20.204/32",
        "52.224.20.223/32",
        "52.224.20.227/32",
        "52.224.20.249/32",
        "52.224.21.19/32",
        "52.224.21.20/32",
        "52.224.21.23/32",
        "52.224.21.27/32",
        "52.224.21.4/32",
        "52.224.21.49/32",
        "52.224.21.51/32",
        "52.224.21.53/32",
        "52.224.21.55/32",
        "52.224.21.61/32",
        "52.242.224.168/32",
        "52.250.46.221/32",
        "57.151.44.140/32",
        "57.152.17.237/32",
        "57.152.62.107/32",
        "57.152.72.128/32",
        "57.155.18.44/32",
        "57.165.205.172/32",
        "57.167.88.38/32",
        "64.236.118.43/32",
        "64.236.15.33/32",
        "64.236.93.100/32",
        "68.219.152.220/32",
        "72.144.95.185/32",
        "74.163.169.106/32",
        "74.163.208.206/32",
        "74.179.200.126/32",
        "74.179.232.116/32",
        "74.242.194.48/32",
        "98.70.219.37/32"
      ]
    },
    {
      "id": "exa-searchbot",
      "name": "ExaSearchBot",
      "operator": {
        "name": "Exa",
        "url": "https://exa.ai"
      },
      "description": "Exa's search crawler, which discovers and indexes pages on the public web so that people and applications can find, retrieve and cite them through Exa. Every request is signed with HTTP Message Signatures (RFC 9421) under the Web Bot Auth scheme, so a request can be verified cryptographically regardless of its source IP address.",
      "docs": [
        "https://crawler.exa.ai/"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "ExaSearchBot/\\d"
        ],
        "instances": [
          "Mozilla/5.0 (compatible; ExaSearchBot/1.0; +https://crawler.exa.ai/)"
        ]
      },
      "behavior": {
        "respects_robots_txt": true,
        "robots_token": "ExaSearchBot"
      },
      "verification": [
        {
          "type": "web_bot_auth",
          "signature_agent_url": "https://crawler.exa.ai/.well-known/http-message-signatures-directory"
        }
      ],
      "tier": 1,
      "tier_label": "Fully verifiable",
      "first_seen": "2026-08-23T13:20:46.873Z",
      "resolved_cidrs": []
    },
    {
      "id": "freespoke-crawler",
      "name": "FreespokeCrawler",
      "operator": {
        "name": "Freespoke",
        "url": "https://freespoke.com"
      },
      "description": "Freespoke's search crawler, which fetches pages to build the index behind the Freespoke search engine. The operator documents the crawler's user agent, the robots.txt token FreespokeCrawler and support for the Crawl-delay directive.",
      "docs": [
        "https://docs.freespoke.com/search/bot/"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "FreespokeCrawler/\\d"
        ],
        "instances": [
          "Mozilla/5.0 (compatible; FreespokeCrawler/2.0; +https://crawler.freespoke.com)"
        ]
      },
      "behavior": {
        "respects_robots_txt": true,
        "robots_token": "FreespokeCrawler"
      },
      "verification": [],
      "tier": 3,
      "tier_label": "Listed only",
      "first_seen": "2026-08-23T13:20:46.873Z",
      "resolved_cidrs": []
    },
    {
      "id": "geedo-crawler-products",
      "name": "Geedo Product Search",
      "operator": {
        "name": "Geedo",
        "url": "https://geedo.com"
      },
      "description": "GeedoShopProductFinder is the crawler for Geedo, a product-search engine that indexes publicly accessible product pages from online stores and follows robots.txt rules.",
      "docs": [
        "https://geedo.com/product-search/"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "GeedoShopProductFinder",
          "GeedoProductSearch"
        ],
        "instances": [
          "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; GeedoShopProductFinder) Chrome/142.0.0.0 Safari/537.36",
          "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GeedoProductSearch; +http://www.geedo.com/product-search.html) Chrome/79.0.3945.88 Safari/537.36"
        ]
      },
      "behavior": {
        "respects_robots_txt": true
      },
      "verification": [
        {
          "type": "cidr_feed",
          "url": "https://geedo.com/product-search/product-search-ip.json",
          "format": "prefixes"
        },
        {
          "type": "rdns",
          "masks": [
            "*.geedo.com"
          ]
        }
      ],
      "tier": 1,
      "tier_label": "Fully verifiable",
      "first_seen": "2026-07-16T00:59:46.504Z",
      "resolved_cidrs": [
        "83.99.206.0/24"
      ]
    },
    {
      "id": "google-crawler-store",
      "name": "Storebot-Google",
      "operator": {
        "name": "Google",
        "url": "https://www.google.com"
      },
      "description": "Google's crawler for Google Shopping surfaces, including the Shopping tab in Google Search and shopping.google.com. It crawls product and store pages to populate shopping results.",
      "docs": [
        "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "Storebot-Google"
        ],
        "instances": [
          "Mozilla/5.0 (X11; Linux x86_64; Storebot-Google/1.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36",
          "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012; Storebot-Google/1.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Mobile Safari/537.36"
        ]
      },
      "behavior": {
        "respects_robots_txt": true,
        "robots_token": "Storebot-Google"
      },
      "verification": [
        {
          "type": "cidr_feed",
          "url": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json",
          "format": "prefixes"
        }
      ],
      "tier": 1,
      "tier_label": "Fully verifiable",
      "first_seen": "2026-07-16T00:59:46.504Z",
      "resolved_cidrs": [
        "192.178.4.0/27",
        "192.178.4.128/27",
        "192.178.4.160/27",
        "192.178.4.192/27",
        "192.178.4.224/27",
        "192.178.4.32/27",
        "192.178.4.64/27",
        "192.178.4.96/27",
        "192.178.5.0/27",
        "192.178.6.0/27",
        "192.178.6.128/27",
        "192.178.6.160/27",
        "192.178.6.192/27",
        "192.178.6.224/27",
        "192.178.6.32/27",
        "192.178.6.64/27",
        "192.178.6.96/27",
        "192.178.7.0/27",
        "192.178.7.128/27",
        "192.178.7.160/27",
        "192.178.7.192/27",
        "192.178.7.224/27",
        "192.178.7.32/27",
        "192.178.7.64/27",
        "192.178.7.96/27",
        "2001:4860:4801:10::/64",
        "2001:4860:4801:11::/64",
        "2001:4860:4801:12::/64",
        "2001:4860:4801:13::/64",
        "2001:4860:4801:14::/64",
        "2001:4860:4801:15::/64",
        "2001:4860:4801:16::/64",
        "2001:4860:4801:17::/64",
        "2001:4860:4801:18::/64",
        "2001:4860:4801:19::/64",
        "2001:4860:4801:1a::/64",
        "2001:4860:4801:1b::/64",
        "2001:4860:4801:1c::/64",
        "2001:4860:4801:1d::/64",
        "2001:4860:4801:1e::/64",
        "2001:4860:4801:1f::/64",
        "2001:4860:4801:20::/64",
        "2001:4860:4801:21::/64",
        "2001:4860:4801:22::/64",
        "2001:4860:4801:23::/64",
        "2001:4860:4801:24::/64",
        "2001:4860:4801:25::/64",
        "2001:4860:4801:26::/64",
        "2001:4860:4801:27::/64",
        "2001:4860:4801:28::/64",
        "2001:4860:4801:29::/64",
        "2001:4860:4801:2::/64",
        "2001:4860:4801:2a::/64",
        "2001:4860:4801:2b::/64",
        "2001:4860:4801:2c::/64",
        "2001:4860:4801:2d::/64",
        "2001:4860:4801:2e::/64",
        "2001:4860:4801:2f::/64",
        "2001:4860:4801:30::/64",
        "2001:4860:4801:31::/64",
        "2001:4860:4801:32::/64",
        "2001:4860:4801:33::/64",
        "2001:4860:4801:34::/64",
        "2001:4860:4801:35::/64",
        "2001:4860:4801:36::/64",
        "2001:4860:4801:37::/64",
        "2001:4860:4801:38::/64",
        "2001:4860:4801:39::/64",
        "2001:4860:4801:3a::/64",
        "2001:4860:4801:3b::/64",
        "2001:4860:4801:3c::/64",
        "2001:4860:4801:3d::/64",
        "2001:4860:4801:3e::/64",
        "2001:4860:4801:3f::/64",
        "2001:4860:4801:40::/64",
        "2001:4860:4801:41::/64",
        "2001:4860:4801:42::/64",
        "2001:4860:4801:44::/64",
        "2001:4860:4801:45::/64",
        "2001:4860:4801:46::/64",
        "2001:4860:4801:47::/64",
        "2001:4860:4801:48::/64",
        "2001:4860:4801:49::/64",
        "2001:4860:4801:4a::/64",
        "2001:4860:4801:4b::/64",
        "2001:4860:4801:4c::/64",
        "2001:4860:4801:4d::/64",
        "2001:4860:4801:4e::/64",
        "2001:4860:4801:50::/64",
        "2001:4860:4801:51::/64",
        "2001:4860:4801:52::/64",
        "2001:4860:4801:53::/64",
        "2001:4860:4801:54::/64",
        "2001:4860:4801:55::/64",
        "2001:4860:4801:56::/64",
        "2001:4860:4801:57::/64",
        "2001:4860:4801:58::/64",
        "2001:4860:4801:59::/64",
        "2001:4860:4801:60::/64",
        "2001:4860:4801:61::/64",
        "2001:4860:4801:62::/64",
        "2001:4860:4801:63::/64",
        "2001:4860:4801:64::/64",
        "2001:4860:4801:65::/64",
        "2001:4860:4801:66::/64",
        "2001:4860:4801:67::/64",
        "2001:4860:4801:68::/64",
        "2001:4860:4801:69::/64",
        "2001:4860:4801:6a::/64",
        "2001:4860:4801:6b::/64",
        "2001:4860:4801:6c::/64",
        "2001:4860:4801:6d::/64",
        "2001:4860:4801:6e::/64",
        "2001:4860:4801:6f::/64",
        "2001:4860:4801:70::/64",
        "2001:4860:4801:71::/64",
        "2001:4860:4801:72::/64",
        "2001:4860:4801:73::/64",
        "2001:4860:4801:74::/64",
        "2001:4860:4801:75::/64",
        "2001:4860:4801:76::/64",
        "2001:4860:4801:77::/64",
        "2001:4860:4801:78::/64",
        "2001:4860:4801:79::/64",
        "2001:4860:4801:7a::/64",
        "2001:4860:4801:7b::/64",
        "2001:4860:4801:7c::/64",
        "2001:4860:4801:7d::/64",
        "2001:4860:4801:7e::/64",
        "2001:4860:4801:7f::/64",
        "2001:4860:4801:80::/64",
        "2001:4860:4801:81::/64",
        "2001:4860:4801:82::/64",
        "2001:4860:4801:83::/64",
        "2001:4860:4801:84::/64",
        "2001:4860:4801:85::/64",
        "2001:4860:4801:86::/64",
        "2001:4860:4801:87::/64",
        "2001:4860:4801:88::/64",
        "2001:4860:4801:90::/64",
        "2001:4860:4801:91::/64",
        "2001:4860:4801:92::/64",
        "2001:4860:4801:93::/64",
        "2001:4860:4801:94::/64",
        "2001:4860:4801:95::/64",
        "2001:4860:4801:96::/64",
        "2001:4860:4801:97::/64",
        "2001:4860:4801:a0::/64",
        "2001:4860:4801:a1::/64",
        "2001:4860:4801:a2::/64",
        "2001:4860:4801:a3::/64",
        "2001:4860:4801:a4::/64",
        "2001:4860:4801:a5::/64",
        "2001:4860:4801:a6::/64",
        "2001:4860:4801:a7::/64",
        "2001:4860:4801:a8::/64",
        "2001:4860:4801:a9::/64",
        "2001:4860:4801:aa::/64",
        "2001:4860:4801:ab::/64",
        "2001:4860:4801:ac::/64",
        "2001:4860:4801:ad::/64",
        "2001:4860:4801:ae::/64",
        "2001:4860:4801:b0::/64",
        "2001:4860:4801:b1::/64",
        "2001:4860:4801:b2::/64",
        "2001:4860:4801:b3::/64",
        "2001:4860:4801:b4::/64",
        "2001:4860:4801:b5::/64",
        "2001:4860:4801:b6::/64",
        "2001:4860:4801:c::/64",
        "2001:4860:4801:f::/64",
        "34.100.182.96/28",
        "34.101.50.144/28",
        "34.118.254.0/28",
        "34.118.66.0/28",
        "34.126.178.96/28",
        "34.146.150.144/28",
        "34.147.110.144/28",
        "34.151.74.144/28",
        "34.152.50.64/28",
        "34.154.114.144/28",
        "34.155.98.32/28",
        "34.165.18.176/28",
        "34.175.160.64/28",
        "34.176.130.16/28",
        "34.22.85.0/27",
        "34.64.82.64/28",
        "34.65.242.112/28",
        "34.80.50.80/28",
        "34.88.194.0/28",
        "34.89.10.80/28",
        "34.89.198.80/28",
        "34.96.162.48/28",
        "35.247.243.240/28",
        "66.249.64.0/27",
        "66.249.64.128/27",
        "66.249.64.160/27",
        "66.249.64.192/27",
        "66.249.64.224/27",
        "66.249.64.32/27",
        "66.249.64.64/27",
        "66.249.64.96/27",
        "66.249.65.0/27",
        "66.249.65.128/27",
        "66.249.65.160/27",
        "66.249.65.192/27",
        "66.249.65.224/27",
        "66.249.65.32/27",
        "66.249.65.64/27",
        "66.249.65.96/27",
        "66.249.66.0/27",
        "66.249.66.128/27",
        "66.249.66.160/27",
        "66.249.66.192/27",
        "66.249.66.224/27",
        "66.249.66.32/27",
        "66.249.66.64/27",
        "66.249.66.96/27",
        "66.249.67.0/27",
        "66.249.67.32/27",
        "66.249.67.64/27",
        "66.249.68.0/27",
        "66.249.68.128/27",
        "66.249.68.160/27",
        "66.249.68.192/27",
        "66.249.68.32/27",
        "66.249.68.64/27",
        "66.249.68.96/27",
        "66.249.69.0/27",
        "66.249.69.128/27",
        "66.249.69.160/27",
        "66.249.69.192/27",
        "66.249.69.224/27",
        "66.249.69.32/27",
        "66.249.69.64/27",
        "66.249.69.96/27",
        "66.249.70.0/27",
        "66.249.70.128/27",
        "66.249.70.160/27",
        "66.249.70.192/27",
        "66.249.70.224/27",
        "66.249.70.32/27",
        "66.249.70.64/27",
        "66.249.70.96/27",
        "66.249.71.0/27",
        "66.249.71.128/27",
        "66.249.71.160/27",
        "66.249.71.192/27",
        "66.249.71.224/27",
        "66.249.71.32/27",
        "66.249.71.64/27",
        "66.249.71.96/27",
        "66.249.72.0/27",
        "66.249.72.128/27",
        "66.249.72.160/27",
        "66.249.72.192/27",
        "66.249.72.224/27",
        "66.249.72.32/27",
        "66.249.72.64/27",
        "66.249.72.96/27",
        "66.249.73.0/27",
        "66.249.73.128/27",
        "66.249.73.160/27",
        "66.249.73.192/27",
        "66.249.73.224/27",
        "66.249.73.32/27",
        "66.249.73.64/27",
        "66.249.73.96/27",
        "66.249.74.0/27",
        "66.249.74.128/27",
        "66.249.74.160/27",
        "66.249.74.192/27",
        "66.249.74.224/27",
        "66.249.74.32/27",
        "66.249.74.64/27",
        "66.249.74.96/27",
        "66.249.75.0/27",
        "66.249.75.128/27",
        "66.249.75.160/27",
        "66.249.75.192/27",
        "66.249.75.224/27",
        "66.249.75.32/27",
        "66.249.75.64/27",
        "66.249.75.96/27",
        "66.249.76.0/27",
        "66.249.76.128/27",
        "66.249.76.160/27",
        "66.249.76.192/27",
        "66.249.76.224/27",
        "66.249.76.32/27",
        "66.249.76.64/27",
        "66.249.76.96/27",
        "66.249.77.0/27",
        "66.249.77.128/27",
        "66.249.77.160/27",
        "66.249.77.192/27",
        "66.249.77.224/27",
        "66.249.77.32/27",
        "66.249.77.64/27",
        "66.249.77.96/27",
        "66.249.78.0/27",
        "66.249.78.128/27",
        "66.249.78.160/27",
        "66.249.78.192/27",
        "66.249.78.224/27",
        "66.249.78.32/27",
        "66.249.78.64/27",
        "66.249.78.96/27",
        "66.249.79.0/27",
        "66.249.79.128/27",
        "66.249.79.160/27",
        "66.249.79.192/27",
        "66.249.79.224/27",
        "66.249.79.32/27",
        "66.249.79.64/27"
      ]
    },
    {
      "id": "google-inspection-tool",
      "name": "Google-InspectionTool",
      "operator": {
        "name": "Google",
        "url": "https://www.google.com"
      },
      "description": "Google crawler used by Search testing tools such as the URL Inspection tool in Search Console and the Rich Result Test. Fetches pages on demand to show how Google Search renders them; it has no effect on ranking.",
      "docs": [
        "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "Google-InspectionTool/\\d"
        ],
        "instances": [
          "Mozilla/5.0 (compatible; Google-InspectionTool/1.0)",
          "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; Google-InspectionTool/1.0)"
        ]
      },
      "behavior": {
        "respects_robots_txt": true,
        "robots_token": "Google-InspectionTool"
      },
      "verification": [
        {
          "type": "cidr_feed",
          "url": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json",
          "format": "prefixes"
        },
        {
          "type": "rdns",
          "masks": [
            "*.googlebot.com",
            "*.geo.googlebot.com"
          ]
        }
      ],
      "tier": 1,
      "tier_label": "Fully verifiable",
      "first_seen": "2026-07-15T01:33:41.731Z",
      "resolved_cidrs": [
        "192.178.4.0/27",
        "192.178.4.128/27",
        "192.178.4.160/27",
        "192.178.4.192/27",
        "192.178.4.224/27",
        "192.178.4.32/27",
        "192.178.4.64/27",
        "192.178.4.96/27",
        "192.178.5.0/27",
        "192.178.6.0/27",
        "192.178.6.128/27",
        "192.178.6.160/27",
        "192.178.6.192/27",
        "192.178.6.224/27",
        "192.178.6.32/27",
        "192.178.6.64/27",
        "192.178.6.96/27",
        "192.178.7.0/27",
        "192.178.7.128/27",
        "192.178.7.160/27",
        "192.178.7.192/27",
        "192.178.7.224/27",
        "192.178.7.32/27",
        "192.178.7.64/27",
        "192.178.7.96/27",
        "2001:4860:4801:10::/64",
        "2001:4860:4801:11::/64",
        "2001:4860:4801:12::/64",
        "2001:4860:4801:13::/64",
        "2001:4860:4801:14::/64",
        "2001:4860:4801:15::/64",
        "2001:4860:4801:16::/64",
        "2001:4860:4801:17::/64",
        "2001:4860:4801:18::/64",
        "2001:4860:4801:19::/64",
        "2001:4860:4801:1a::/64",
        "2001:4860:4801:1b::/64",
        "2001:4860:4801:1c::/64",
        "2001:4860:4801:1d::/64",
        "2001:4860:4801:1e::/64",
        "2001:4860:4801:1f::/64",
        "2001:4860:4801:20::/64",
        "2001:4860:4801:21::/64",
        "2001:4860:4801:22::/64",
        "2001:4860:4801:23::/64",
        "2001:4860:4801:24::/64",
        "2001:4860:4801:25::/64",
        "2001:4860:4801:26::/64",
        "2001:4860:4801:27::/64",
        "2001:4860:4801:28::/64",
        "2001:4860:4801:29::/64",
        "2001:4860:4801:2::/64",
        "2001:4860:4801:2a::/64",
        "2001:4860:4801:2b::/64",
        "2001:4860:4801:2c::/64",
        "2001:4860:4801:2d::/64",
        "2001:4860:4801:2e::/64",
        "2001:4860:4801:2f::/64",
        "2001:4860:4801:30::/64",
        "2001:4860:4801:31::/64",
        "2001:4860:4801:32::/64",
        "2001:4860:4801:33::/64",
        "2001:4860:4801:34::/64",
        "2001:4860:4801:35::/64",
        "2001:4860:4801:36::/64",
        "2001:4860:4801:37::/64",
        "2001:4860:4801:38::/64",
        "2001:4860:4801:39::/64",
        "2001:4860:4801:3a::/64",
        "2001:4860:4801:3b::/64",
        "2001:4860:4801:3c::/64",
        "2001:4860:4801:3d::/64",
        "2001:4860:4801:3e::/64",
        "2001:4860:4801:3f::/64",
        "2001:4860:4801:40::/64",
        "2001:4860:4801:41::/64",
        "2001:4860:4801:42::/64",
        "2001:4860:4801:44::/64",
        "2001:4860:4801:45::/64",
        "2001:4860:4801:46::/64",
        "2001:4860:4801:47::/64",
        "2001:4860:4801:48::/64",
        "2001:4860:4801:49::/64",
        "2001:4860:4801:4a::/64",
        "2001:4860:4801:4b::/64",
        "2001:4860:4801:4c::/64",
        "2001:4860:4801:4d::/64",
        "2001:4860:4801:4e::/64",
        "2001:4860:4801:50::/64",
        "2001:4860:4801:51::/64",
        "2001:4860:4801:52::/64",
        "2001:4860:4801:53::/64",
        "2001:4860:4801:54::/64",
        "2001:4860:4801:55::/64",
        "2001:4860:4801:56::/64",
        "2001:4860:4801:57::/64",
        "2001:4860:4801:58::/64",
        "2001:4860:4801:59::/64",
        "2001:4860:4801:60::/64",
        "2001:4860:4801:61::/64",
        "2001:4860:4801:62::/64",
        "2001:4860:4801:63::/64",
        "2001:4860:4801:64::/64",
        "2001:4860:4801:65::/64",
        "2001:4860:4801:66::/64",
        "2001:4860:4801:67::/64",
        "2001:4860:4801:68::/64",
        "2001:4860:4801:69::/64",
        "2001:4860:4801:6a::/64",
        "2001:4860:4801:6b::/64",
        "2001:4860:4801:6c::/64",
        "2001:4860:4801:6d::/64",
        "2001:4860:4801:6e::/64",
        "2001:4860:4801:6f::/64",
        "2001:4860:4801:70::/64",
        "2001:4860:4801:71::/64",
        "2001:4860:4801:72::/64",
        "2001:4860:4801:73::/64",
        "2001:4860:4801:74::/64",
        "2001:4860:4801:75::/64",
        "2001:4860:4801:76::/64",
        "2001:4860:4801:77::/64",
        "2001:4860:4801:78::/64",
        "2001:4860:4801:79::/64",
        "2001:4860:4801:7a::/64",
        "2001:4860:4801:7b::/64",
        "2001:4860:4801:7c::/64",
        "2001:4860:4801:7d::/64",
        "2001:4860:4801:7e::/64",
        "2001:4860:4801:7f::/64",
        "2001:4860:4801:80::/64",
        "2001:4860:4801:81::/64",
        "2001:4860:4801:82::/64",
        "2001:4860:4801:83::/64",
        "2001:4860:4801:84::/64",
        "2001:4860:4801:85::/64",
        "2001:4860:4801:86::/64",
        "2001:4860:4801:87::/64",
        "2001:4860:4801:88::/64",
        "2001:4860:4801:90::/64",
        "2001:4860:4801:91::/64",
        "2001:4860:4801:92::/64",
        "2001:4860:4801:93::/64",
        "2001:4860:4801:94::/64",
        "2001:4860:4801:95::/64",
        "2001:4860:4801:96::/64",
        "2001:4860:4801:97::/64",
        "2001:4860:4801:a0::/64",
        "2001:4860:4801:a1::/64",
        "2001:4860:4801:a2::/64",
        "2001:4860:4801:a3::/64",
        "2001:4860:4801:a4::/64",
        "2001:4860:4801:a5::/64",
        "2001:4860:4801:a6::/64",
        "2001:4860:4801:a7::/64",
        "2001:4860:4801:a8::/64",
        "2001:4860:4801:a9::/64",
        "2001:4860:4801:aa::/64",
        "2001:4860:4801:ab::/64",
        "2001:4860:4801:ac::/64",
        "2001:4860:4801:ad::/64",
        "2001:4860:4801:ae::/64",
        "2001:4860:4801:b0::/64",
        "2001:4860:4801:b1::/64",
        "2001:4860:4801:b2::/64",
        "2001:4860:4801:b3::/64",
        "2001:4860:4801:b4::/64",
        "2001:4860:4801:b5::/64",
        "2001:4860:4801:b6::/64",
        "2001:4860:4801:c::/64",
        "2001:4860:4801:f::/64",
        "34.100.182.96/28",
        "34.101.50.144/28",
        "34.118.254.0/28",
        "34.118.66.0/28",
        "34.126.178.96/28",
        "34.146.150.144/28",
        "34.147.110.144/28",
        "34.151.74.144/28",
        "34.152.50.64/28",
        "34.154.114.144/28",
        "34.155.98.32/28",
        "34.165.18.176/28",
        "34.175.160.64/28",
        "34.176.130.16/28",
        "34.22.85.0/27",
        "34.64.82.64/28",
        "34.65.242.112/28",
        "34.80.50.80/28",
        "34.88.194.0/28",
        "34.89.10.80/28",
        "34.89.198.80/28",
        "34.96.162.48/28",
        "35.247.243.240/28",
        "66.249.64.0/27",
        "66.249.64.128/27",
        "66.249.64.160/27",
        "66.249.64.192/27",
        "66.249.64.224/27",
        "66.249.64.32/27",
        "66.249.64.64/27",
        "66.249.64.96/27",
        "66.249.65.0/27",
        "66.249.65.128/27",
        "66.249.65.160/27",
        "66.249.65.192/27",
        "66.249.65.224/27",
        "66.249.65.32/27",
        "66.249.65.64/27",
        "66.249.65.96/27",
        "66.249.66.0/27",
        "66.249.66.128/27",
        "66.249.66.160/27",
        "66.249.66.192/27",
        "66.249.66.224/27",
        "66.249.66.32/27",
        "66.249.66.64/27",
        "66.249.66.96/27",
        "66.249.67.0/27",
        "66.249.67.32/27",
        "66.249.67.64/27",
        "66.249.68.0/27",
        "66.249.68.128/27",
        "66.249.68.160/27",
        "66.249.68.192/27",
        "66.249.68.32/27",
        "66.249.68.64/27",
        "66.249.68.96/27",
        "66.249.69.0/27",
        "66.249.69.128/27",
        "66.249.69.160/27",
        "66.249.69.192/27",
        "66.249.69.224/27",
        "66.249.69.32/27",
        "66.249.69.64/27",
        "66.249.69.96/27",
        "66.249.70.0/27",
        "66.249.70.128/27",
        "66.249.70.160/27",
        "66.249.70.192/27",
        "66.249.70.224/27",
        "66.249.70.32/27",
        "66.249.70.64/27",
        "66.249.70.96/27",
        "66.249.71.0/27",
        "66.249.71.128/27",
        "66.249.71.160/27",
        "66.249.71.192/27",
        "66.249.71.224/27",
        "66.249.71.32/27",
        "66.249.71.64/27",
        "66.249.71.96/27",
        "66.249.72.0/27",
        "66.249.72.128/27",
        "66.249.72.160/27",
        "66.249.72.192/27",
        "66.249.72.224/27",
        "66.249.72.32/27",
        "66.249.72.64/27",
        "66.249.72.96/27",
        "66.249.73.0/27",
        "66.249.73.128/27",
        "66.249.73.160/27",
        "66.249.73.192/27",
        "66.249.73.224/27",
        "66.249.73.32/27",
        "66.249.73.64/27",
        "66.249.73.96/27",
        "66.249.74.0/27",
        "66.249.74.128/27",
        "66.249.74.160/27",
        "66.249.74.192/27",
        "66.249.74.224/27",
        "66.249.74.32/27",
        "66.249.74.64/27",
        "66.249.74.96/27",
        "66.249.75.0/27",
        "66.249.75.128/27",
        "66.249.75.160/27",
        "66.249.75.192/27",
        "66.249.75.224/27",
        "66.249.75.32/27",
        "66.249.75.64/27",
        "66.249.75.96/27",
        "66.249.76.0/27",
        "66.249.76.128/27",
        "66.249.76.160/27",
        "66.249.76.192/27",
        "66.249.76.224/27",
        "66.249.76.32/27",
        "66.249.76.64/27",
        "66.249.76.96/27",
        "66.249.77.0/27",
        "66.249.77.128/27",
        "66.249.77.160/27",
        "66.249.77.192/27",
        "66.249.77.224/27",
        "66.249.77.32/27",
        "66.249.77.64/27",
        "66.249.77.96/27",
        "66.249.78.0/27",
        "66.249.78.128/27",
        "66.249.78.160/27",
        "66.249.78.192/27",
        "66.249.78.224/27",
        "66.249.78.32/27",
        "66.249.78.64/27",
        "66.249.78.96/27",
        "66.249.79.0/27",
        "66.249.79.128/27",
        "66.249.79.160/27",
        "66.249.79.192/27",
        "66.249.79.224/27",
        "66.249.79.32/27",
        "66.249.79.64/27"
      ]
    },
    {
      "id": "google-special-crawlers",
      "name": "Google special-case crawlers",
      "operator": {
        "name": "Google",
        "url": "https://www.google.com"
      },
      "description": "A group of Google crawlers that operate under separate agreements between the crawled site and a specific Google product (AdsBot, AdSense, Google Safety scanning, and similar). Several members of this group ignore the wildcard robots.txt rule and must be targeted explicitly.",
      "docs": [
        "https://developers.google.com/crawling/docs/crawlers-fetchers/google-special-case-crawlers"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "AdsBot-Google",
          "Mediapartners-Google",
          "Google-Safety"
        ],
        "instances": [
          "AdsBot-Google (+http://www.google.com/adsbot.html)",
          "Mediapartners-Google",
          "Google-Safety"
        ]
      },
      "behavior": {
        "respects_robots_txt": false
      },
      "verification": [
        {
          "type": "cidr_feed",
          "url": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json",
          "format": "prefixes"
        }
      ],
      "tier": 1,
      "tier_label": "Fully verifiable",
      "first_seen": "2026-07-08T19:13:49.571Z",
      "resolved_cidrs": [
        "108.177.2.0/27",
        "108.177.2.128/27",
        "108.177.2.160/27",
        "108.177.2.192/27",
        "108.177.2.224/27",
        "108.177.2.32/27",
        "108.177.2.64/27",
        "108.177.2.96/27",
        "192.178.16.0/27",
        "192.178.16.128/27",
        "192.178.16.160/27",
        "192.178.16.192/27",
        "192.178.16.224/27",
        "192.178.16.32/27",
        "192.178.16.64/27",
        "192.178.16.96/27",
        "192.178.17.0/27",
        "2001:4860:4801:2008::/64",
        "2001:4860:4801:200c::/64",
        "2001:4860:4801:200d::/64",
        "2001:4860:4801:2010::/64",
        "2001:4860:4801:2011::/64",
        "2001:4860:4801:2012::/64",
        "2001:4860:4801:2013::/64",
        "2001:4860:4801:2014::/64",
        "2001:4860:4801:2015::/64",
        "2001:4860:4801:2016::/64",
        "2001:4860:4801:2017::/64",
        "2001:4860:4801:2018::/64",
        "2001:4860:4801:2019::/64",
        "2001:4860:4801:201a::/64",
        "2001:4860:4801:201b::/64",
        "2001:4860:4801:201c::/64",
        "2001:4860:4801:201d::/64",
        "2001:4860:4801:201e::/64",
        "2001:4860:4801:201f::/64",
        "2001:4860:4801:2020::/64",
        "2001:4860:4801:2021::/64",
        "2001:4860:4801:2022::/64",
        "2001:4860:4801:2023::/64",
        "2001:4860:4801:2024::/64",
        "2001:4860:4801:2025::/64",
        "2001:4860:4801:2026::/64",
        "2001:4860:4801:2027::/64",
        "2001:4860:4801:2028::/64",
        "2001:4860:4801:2029::/64",
        "2001:4860:4801:202a::/64",
        "2001:4860:4801:202b::/64",
        "2001:4860:4801:202c::/64",
        "2001:4860:4801:202d::/64",
        "2001:4860:4801:202e::/64",
        "2001:4860:4801:202f::/64",
        "2001:4860:4801:2030::/64",
        "2001:4860:4801:2031::/64",
        "2001:4860:4801:2032::/64",
        "2001:4860:4801:2033::/64",
        "2001:4860:4801:2034::/64",
        "2001:4860:4801:2035::/64",
        "2001:4860:4801:2036::/64",
        "2001:4860:4801:2037::/64",
        "2001:4860:4801:2038::/64",
        "2001:4860:4801:2039::/64",
        "2001:4860:4801:203a::/64",
        "2001:4860:4801:203b::/64",
        "2001:4860:4801:203c::/64",
        "2001:4860:4801:203d::/64",
        "2001:4860:4801:203e::/64",
        "2001:4860:4801:203f::/64",
        "2001:4860:4801:2040::/64",
        "2001:4860:4801:2041::/64",
        "2001:4860:4801:2042::/64",
        "2001:4860:4801:2044::/64",
        "2001:4860:4801:2045::/64",
        "2001:4860:4801:2046::/64",
        "2001:4860:4801:2047::/64",
        "2001:4860:4801:2048::/64",
        "2001:4860:4801:2049::/64",
        "2001:4860:4801:204a::/64",
        "2001:4860:4801:204b::/64",
        "2001:4860:4801:204c::/64",
        "2001:4860:4801:204d::/64",
        "2001:4860:4801:204e::/64",
        "2001:4860:4801:2050::/64",
        "2001:4860:4801:2051::/64",
        "2001:4860:4801:2052::/64",
        "2001:4860:4801:2053::/64",
        "2001:4860:4801:2054::/64",
        "2001:4860:4801:2055::/64",
        "2001:4860:4801:2056::/64",
        "2001:4860:4801:2060::/64",
        "2001:4860:4801:2061::/64",
        "2001:4860:4801:2062::/64",
        "2001:4860:4801:2063::/64",
        "2001:4860:4801:2064::/64",
        "2001:4860:4801:2065::/64",
        "2001:4860:4801:2066::/64",
        "2001:4860:4801:2067::/64",
        "2001:4860:4801:2068::/64",
        "2001:4860:4801:2069::/64",
        "2001:4860:4801:206a::/64",
        "2001:4860:4801:206b::/64",
        "2001:4860:4801:206c::/64",
        "2001:4860:4801:206d::/64",
        "2001:4860:4801:206e::/64",
        "2001:4860:4801:206f::/64",
        "2001:4860:4801:2070::/64",
        "2001:4860:4801:2071::/64",
        "2001:4860:4801:2072::/64",
        "2001:4860:4801:2073::/64",
        "2001:4860:4801:2074::/64",
        "2001:4860:4801:2075::/64",
        "2001:4860:4801:2076::/64",
        "2001:4860:4801:2077::/64",
        "2001:4860:4801:2078::/64",
        "2001:4860:4801:2079::/64",
        "2001:4860:4801:207a::/64",
        "2001:4860:4801:207b::/64",
        "2001:4860:4801:207c::/64",
        "2001:4860:4801:207d::/64",
        "2001:4860:4801:207e::/64",
        "2001:4860:4801:207f::/64",
        "2001:4860:4801:2080::/64",
        "2001:4860:4801:2081::/64",
        "2001:4860:4801:2082::/64",
        "2001:4860:4801:2083::/64",
        "2001:4860:4801:2084::/64",
        "2001:4860:4801:2085::/64",
        "2001:4860:4801:2086::/64",
        "2001:4860:4801:2087::/64",
        "2001:4860:4801:2088::/64",
        "2001:4860:4801:2090::/64",
        "2001:4860:4801:2091::/64",
        "2001:4860:4801:2092::/64",
        "2001:4860:4801:2093::/64",
        "2001:4860:4801:2094::/64",
        "2001:4860:4801:2095::/64",
        "2001:4860:4801:2096::/64",
        "2001:4860:4801:2097::/64",
        "2001:4860:4801:20a0::/64",
        "2001:4860:4801:20a1::/64",
        "2001:4860:4801:20a2::/64",
        "2001:4860:4801:20a3::/64",
        "2001:4860:4801:20a4::/64",
        "2001:4860:4801:20a5::/64",
        "2001:4860:4801:20a6::/64",
        "2001:4860:4801:20b0::/64",
        "2001:4860:4801:20b1::/64",
        "2001:4860:4801:20b2::/64",
        "2001:4860:4801:20b3::/64",
        "2001:4860:4801:20b4::/64",
        "2001:4860:4801:20b5::/64",
        "2001:4860:4801:20b6::/64",
        "209.85.238.0/27",
        "209.85.238.128/27",
        "209.85.238.160/27",
        "209.85.238.192/27",
        "209.85.238.224/27",
        "209.85.238.32/27",
        "209.85.238.64/27",
        "209.85.238.96/27",
        "66.249.87.0/27",
        "66.249.87.128/27",
        "66.249.87.160/27",
        "66.249.87.192/27",
        "66.249.87.224/27",
        "66.249.87.32/27",
        "66.249.87.64/27",
        "66.249.87.96/27",
        "66.249.89.0/27",
        "66.249.89.128/27",
        "66.249.89.160/27",
        "66.249.89.192/27",
        "66.249.89.224/27",
        "66.249.89.32/27",
        "66.249.89.64/27",
        "66.249.89.96/27",
        "66.249.90.0/27",
        "66.249.90.128/27",
        "66.249.90.160/27",
        "66.249.90.192/27",
        "66.249.90.224/27",
        "66.249.90.32/27",
        "66.249.90.64/27",
        "66.249.90.96/27",
        "66.249.91.0/27",
        "66.249.91.128/27",
        "66.249.91.160/27",
        "66.249.91.192/27",
        "66.249.91.224/27",
        "66.249.91.32/27",
        "66.249.91.64/27",
        "66.249.91.96/27",
        "66.249.92.0/27",
        "66.249.92.128/27",
        "66.249.92.160/27",
        "66.249.92.192/27",
        "66.249.92.224/27",
        "66.249.92.32/27",
        "66.249.92.64/27",
        "66.249.92.96/27",
        "72.14.199.128/27",
        "72.14.199.160/27",
        "72.14.199.192/27",
        "72.14.199.224/27",
        "72.14.199.32/27",
        "72.14.199.64/27",
        "72.14.199.96/27",
        "74.125.148.0/27",
        "74.125.148.128/27",
        "74.125.148.160/27",
        "74.125.148.192/27",
        "74.125.148.224/27",
        "74.125.148.32/27",
        "74.125.148.64/27",
        "74.125.148.96/27",
        "74.125.149.0/27",
        "74.125.149.128/27",
        "74.125.149.160/27",
        "74.125.149.192/27",
        "74.125.149.224/27",
        "74.125.149.32/27",
        "74.125.149.64/27",
        "74.125.149.96/27",
        "74.125.150.0/27",
        "74.125.150.128/27",
        "74.125.150.160/27",
        "74.125.150.192/27",
        "74.125.150.32/27",
        "74.125.150.64/27",
        "74.125.150.96/27",
        "74.125.151.0/27",
        "74.125.151.128/27",
        "74.125.151.160/27",
        "74.125.151.192/27",
        "74.125.151.224/27",
        "74.125.151.32/27",
        "74.125.151.64/27",
        "74.125.151.96/27",
        "74.125.216.0/27",
        "74.125.216.128/27",
        "74.125.216.160/27",
        "74.125.216.192/27",
        "74.125.216.224/27",
        "74.125.216.32/27",
        "74.125.216.64/27",
        "74.125.216.96/27",
        "74.125.217.0/27",
        "74.125.217.128/27",
        "74.125.217.160/27",
        "74.125.217.192/27",
        "74.125.217.224/27",
        "74.125.217.32/27",
        "74.125.217.64/27",
        "74.125.217.96/27",
        "74.125.218.0/27",
        "74.125.218.128/27",
        "74.125.218.160/27",
        "74.125.218.192/27",
        "74.125.218.224/27",
        "74.125.218.32/27",
        "74.125.218.64/27",
        "74.125.218.96/27",
        "74.125.219.0/27",
        "74.125.219.128/27",
        "74.125.219.160/27",
        "74.125.219.192/27",
        "74.125.219.224/27",
        "74.125.219.32/27",
        "74.125.219.64/27",
        "74.125.219.96/27"
      ]
    },
    {
      "id": "googlebot",
      "name": "Googlebot",
      "operator": {
        "name": "Google",
        "url": "https://www.google.com"
      },
      "description": "Google's primary web crawler, fetching pages for Google Search indexing.",
      "docs": [
        "https://developers.google.com/search/docs/crawling-indexing/verifying-googlebot"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "Googlebot/\\d"
        ],
        "instances": [
          "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/125.0.6422.26 Safari/537.36",
          "Googlebot/2.1 (+http://www.google.com/bot.html)"
        ]
      },
      "behavior": {
        "respects_robots_txt": true,
        "robots_token": "Googlebot"
      },
      "verification": [
        {
          "type": "cidr_feed",
          "url": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json",
          "format": "prefixes"
        },
        {
          "type": "rdns",
          "masks": [
            "*.googlebot.com",
            "*.geo.googlebot.com"
          ]
        }
      ],
      "tier": 1,
      "tier_label": "Fully verifiable",
      "first_seen": "2026-07-08T19:13:49.571Z",
      "resolved_cidrs": [
        "192.178.4.0/27",
        "192.178.4.128/27",
        "192.178.4.160/27",
        "192.178.4.192/27",
        "192.178.4.224/27",
        "192.178.4.32/27",
        "192.178.4.64/27",
        "192.178.4.96/27",
        "192.178.5.0/27",
        "192.178.6.0/27",
        "192.178.6.128/27",
        "192.178.6.160/27",
        "192.178.6.192/27",
        "192.178.6.224/27",
        "192.178.6.32/27",
        "192.178.6.64/27",
        "192.178.6.96/27",
        "192.178.7.0/27",
        "192.178.7.128/27",
        "192.178.7.160/27",
        "192.178.7.192/27",
        "192.178.7.224/27",
        "192.178.7.32/27",
        "192.178.7.64/27",
        "192.178.7.96/27",
        "2001:4860:4801:10::/64",
        "2001:4860:4801:11::/64",
        "2001:4860:4801:12::/64",
        "2001:4860:4801:13::/64",
        "2001:4860:4801:14::/64",
        "2001:4860:4801:15::/64",
        "2001:4860:4801:16::/64",
        "2001:4860:4801:17::/64",
        "2001:4860:4801:18::/64",
        "2001:4860:4801:19::/64",
        "2001:4860:4801:1a::/64",
        "2001:4860:4801:1b::/64",
        "2001:4860:4801:1c::/64",
        "2001:4860:4801:1d::/64",
        "2001:4860:4801:1e::/64",
        "2001:4860:4801:1f::/64",
        "2001:4860:4801:20::/64",
        "2001:4860:4801:21::/64",
        "2001:4860:4801:22::/64",
        "2001:4860:4801:23::/64",
        "2001:4860:4801:24::/64",
        "2001:4860:4801:25::/64",
        "2001:4860:4801:26::/64",
        "2001:4860:4801:27::/64",
        "2001:4860:4801:28::/64",
        "2001:4860:4801:29::/64",
        "2001:4860:4801:2::/64",
        "2001:4860:4801:2a::/64",
        "2001:4860:4801:2b::/64",
        "2001:4860:4801:2c::/64",
        "2001:4860:4801:2d::/64",
        "2001:4860:4801:2e::/64",
        "2001:4860:4801:2f::/64",
        "2001:4860:4801:30::/64",
        "2001:4860:4801:31::/64",
        "2001:4860:4801:32::/64",
        "2001:4860:4801:33::/64",
        "2001:4860:4801:34::/64",
        "2001:4860:4801:35::/64",
        "2001:4860:4801:36::/64",
        "2001:4860:4801:37::/64",
        "2001:4860:4801:38::/64",
        "2001:4860:4801:39::/64",
        "2001:4860:4801:3a::/64",
        "2001:4860:4801:3b::/64",
        "2001:4860:4801:3c::/64",
        "2001:4860:4801:3d::/64",
        "2001:4860:4801:3e::/64",
        "2001:4860:4801:3f::/64",
        "2001:4860:4801:40::/64",
        "2001:4860:4801:41::/64",
        "2001:4860:4801:42::/64",
        "2001:4860:4801:44::/64",
        "2001:4860:4801:45::/64",
        "2001:4860:4801:46::/64",
        "2001:4860:4801:47::/64",
        "2001:4860:4801:48::/64",
        "2001:4860:4801:49::/64",
        "2001:4860:4801:4a::/64",
        "2001:4860:4801:4b::/64",
        "2001:4860:4801:4c::/64",
        "2001:4860:4801:4d::/64",
        "2001:4860:4801:4e::/64",
        "2001:4860:4801:50::/64",
        "2001:4860:4801:51::/64",
        "2001:4860:4801:52::/64",
        "2001:4860:4801:53::/64",
        "2001:4860:4801:54::/64",
        "2001:4860:4801:55::/64",
        "2001:4860:4801:56::/64",
        "2001:4860:4801:57::/64",
        "2001:4860:4801:58::/64",
        "2001:4860:4801:59::/64",
        "2001:4860:4801:60::/64",
        "2001:4860:4801:61::/64",
        "2001:4860:4801:62::/64",
        "2001:4860:4801:63::/64",
        "2001:4860:4801:64::/64",
        "2001:4860:4801:65::/64",
        "2001:4860:4801:66::/64",
        "2001:4860:4801:67::/64",
        "2001:4860:4801:68::/64",
        "2001:4860:4801:69::/64",
        "2001:4860:4801:6a::/64",
        "2001:4860:4801:6b::/64",
        "2001:4860:4801:6c::/64",
        "2001:4860:4801:6d::/64",
        "2001:4860:4801:6e::/64",
        "2001:4860:4801:6f::/64",
        "2001:4860:4801:70::/64",
        "2001:4860:4801:71::/64",
        "2001:4860:4801:72::/64",
        "2001:4860:4801:73::/64",
        "2001:4860:4801:74::/64",
        "2001:4860:4801:75::/64",
        "2001:4860:4801:76::/64",
        "2001:4860:4801:77::/64",
        "2001:4860:4801:78::/64",
        "2001:4860:4801:79::/64",
        "2001:4860:4801:7a::/64",
        "2001:4860:4801:7b::/64",
        "2001:4860:4801:7c::/64",
        "2001:4860:4801:7d::/64",
        "2001:4860:4801:7e::/64",
        "2001:4860:4801:7f::/64",
        "2001:4860:4801:80::/64",
        "2001:4860:4801:81::/64",
        "2001:4860:4801:82::/64",
        "2001:4860:4801:83::/64",
        "2001:4860:4801:84::/64",
        "2001:4860:4801:85::/64",
        "2001:4860:4801:86::/64",
        "2001:4860:4801:87::/64",
        "2001:4860:4801:88::/64",
        "2001:4860:4801:90::/64",
        "2001:4860:4801:91::/64",
        "2001:4860:4801:92::/64",
        "2001:4860:4801:93::/64",
        "2001:4860:4801:94::/64",
        "2001:4860:4801:95::/64",
        "2001:4860:4801:96::/64",
        "2001:4860:4801:97::/64",
        "2001:4860:4801:a0::/64",
        "2001:4860:4801:a1::/64",
        "2001:4860:4801:a2::/64",
        "2001:4860:4801:a3::/64",
        "2001:4860:4801:a4::/64",
        "2001:4860:4801:a5::/64",
        "2001:4860:4801:a6::/64",
        "2001:4860:4801:a7::/64",
        "2001:4860:4801:a8::/64",
        "2001:4860:4801:a9::/64",
        "2001:4860:4801:aa::/64",
        "2001:4860:4801:ab::/64",
        "2001:4860:4801:ac::/64",
        "2001:4860:4801:ad::/64",
        "2001:4860:4801:ae::/64",
        "2001:4860:4801:b0::/64",
        "2001:4860:4801:b1::/64",
        "2001:4860:4801:b2::/64",
        "2001:4860:4801:b3::/64",
        "2001:4860:4801:b4::/64",
        "2001:4860:4801:b5::/64",
        "2001:4860:4801:b6::/64",
        "2001:4860:4801:c::/64",
        "2001:4860:4801:f::/64",
        "34.100.182.96/28",
        "34.101.50.144/28",
        "34.118.254.0/28",
        "34.118.66.0/28",
        "34.126.178.96/28",
        "34.146.150.144/28",
        "34.147.110.144/28",
        "34.151.74.144/28",
        "34.152.50.64/28",
        "34.154.114.144/28",
        "34.155.98.32/28",
        "34.165.18.176/28",
        "34.175.160.64/28",
        "34.176.130.16/28",
        "34.22.85.0/27",
        "34.64.82.64/28",
        "34.65.242.112/28",
        "34.80.50.80/28",
        "34.88.194.0/28",
        "34.89.10.80/28",
        "34.89.198.80/28",
        "34.96.162.48/28",
        "35.247.243.240/28",
        "66.249.64.0/27",
        "66.249.64.128/27",
        "66.249.64.160/27",
        "66.249.64.192/27",
        "66.249.64.224/27",
        "66.249.64.32/27",
        "66.249.64.64/27",
        "66.249.64.96/27",
        "66.249.65.0/27",
        "66.249.65.128/27",
        "66.249.65.160/27",
        "66.249.65.192/27",
        "66.249.65.224/27",
        "66.249.65.32/27",
        "66.249.65.64/27",
        "66.249.65.96/27",
        "66.249.66.0/27",
        "66.249.66.128/27",
        "66.249.66.160/27",
        "66.249.66.192/27",
        "66.249.66.224/27",
        "66.249.66.32/27",
        "66.249.66.64/27",
        "66.249.66.96/27",
        "66.249.67.0/27",
        "66.249.67.32/27",
        "66.249.67.64/27",
        "66.249.68.0/27",
        "66.249.68.128/27",
        "66.249.68.160/27",
        "66.249.68.192/27",
        "66.249.68.32/27",
        "66.249.68.64/27",
        "66.249.68.96/27",
        "66.249.69.0/27",
        "66.249.69.128/27",
        "66.249.69.160/27",
        "66.249.69.192/27",
        "66.249.69.224/27",
        "66.249.69.32/27",
        "66.249.69.64/27",
        "66.249.69.96/27",
        "66.249.70.0/27",
        "66.249.70.128/27",
        "66.249.70.160/27",
        "66.249.70.192/27",
        "66.249.70.224/27",
        "66.249.70.32/27",
        "66.249.70.64/27",
        "66.249.70.96/27",
        "66.249.71.0/27",
        "66.249.71.128/27",
        "66.249.71.160/27",
        "66.249.71.192/27",
        "66.249.71.224/27",
        "66.249.71.32/27",
        "66.249.71.64/27",
        "66.249.71.96/27",
        "66.249.72.0/27",
        "66.249.72.128/27",
        "66.249.72.160/27",
        "66.249.72.192/27",
        "66.249.72.224/27",
        "66.249.72.32/27",
        "66.249.72.64/27",
        "66.249.72.96/27",
        "66.249.73.0/27",
        "66.249.73.128/27",
        "66.249.73.160/27",
        "66.249.73.192/27",
        "66.249.73.224/27",
        "66.249.73.32/27",
        "66.249.73.64/27",
        "66.249.73.96/27",
        "66.249.74.0/27",
        "66.249.74.128/27",
        "66.249.74.160/27",
        "66.249.74.192/27",
        "66.249.74.224/27",
        "66.249.74.32/27",
        "66.249.74.64/27",
        "66.249.74.96/27",
        "66.249.75.0/27",
        "66.249.75.128/27",
        "66.249.75.160/27",
        "66.249.75.192/27",
        "66.249.75.224/27",
        "66.249.75.32/27",
        "66.249.75.64/27",
        "66.249.75.96/27",
        "66.249.76.0/27",
        "66.249.76.128/27",
        "66.249.76.160/27",
        "66.249.76.192/27",
        "66.249.76.224/27",
        "66.249.76.32/27",
        "66.249.76.64/27",
        "66.249.76.96/27",
        "66.249.77.0/27",
        "66.249.77.128/27",
        "66.249.77.160/27",
        "66.249.77.192/27",
        "66.249.77.224/27",
        "66.249.77.32/27",
        "66.249.77.64/27",
        "66.249.77.96/27",
        "66.249.78.0/27",
        "66.249.78.128/27",
        "66.249.78.160/27",
        "66.249.78.192/27",
        "66.249.78.224/27",
        "66.249.78.32/27",
        "66.249.78.64/27",
        "66.249.78.96/27",
        "66.249.79.0/27",
        "66.249.79.128/27",
        "66.249.79.160/27",
        "66.249.79.192/27",
        "66.249.79.224/27",
        "66.249.79.32/27",
        "66.249.79.64/27"
      ]
    },
    {
      "id": "googlebot-image",
      "name": "Googlebot Image",
      "operator": {
        "name": "Google",
        "url": "https://www.google.com"
      },
      "description": "Google's image crawler, which fetches images for Google Images and image features in Search. It is a distinct crawler from Googlebot with its own robots.txt token, though rules for Googlebot also apply to it.",
      "docs": [
        "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "Googlebot-Image/\\d"
        ],
        "instances": [
          "Googlebot-Image/1.0"
        ]
      },
      "behavior": {
        "respects_robots_txt": true,
        "robots_token": "Googlebot-Image"
      },
      "verification": [
        {
          "type": "cidr_feed",
          "url": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json",
          "format": "prefixes"
        },
        {
          "type": "rdns",
          "masks": [
            "*.googlebot.com",
            "*.geo.googlebot.com"
          ]
        }
      ],
      "tier": 1,
      "tier_label": "Fully verifiable",
      "first_seen": "2026-07-18T15:23:43.605Z",
      "resolved_cidrs": [
        "192.178.4.0/27",
        "192.178.4.128/27",
        "192.178.4.160/27",
        "192.178.4.192/27",
        "192.178.4.224/27",
        "192.178.4.32/27",
        "192.178.4.64/27",
        "192.178.4.96/27",
        "192.178.5.0/27",
        "192.178.6.0/27",
        "192.178.6.128/27",
        "192.178.6.160/27",
        "192.178.6.192/27",
        "192.178.6.224/27",
        "192.178.6.32/27",
        "192.178.6.64/27",
        "192.178.6.96/27",
        "192.178.7.0/27",
        "192.178.7.128/27",
        "192.178.7.160/27",
        "192.178.7.192/27",
        "192.178.7.224/27",
        "192.178.7.32/27",
        "192.178.7.64/27",
        "192.178.7.96/27",
        "2001:4860:4801:10::/64",
        "2001:4860:4801:11::/64",
        "2001:4860:4801:12::/64",
        "2001:4860:4801:13::/64",
        "2001:4860:4801:14::/64",
        "2001:4860:4801:15::/64",
        "2001:4860:4801:16::/64",
        "2001:4860:4801:17::/64",
        "2001:4860:4801:18::/64",
        "2001:4860:4801:19::/64",
        "2001:4860:4801:1a::/64",
        "2001:4860:4801:1b::/64",
        "2001:4860:4801:1c::/64",
        "2001:4860:4801:1d::/64",
        "2001:4860:4801:1e::/64",
        "2001:4860:4801:1f::/64",
        "2001:4860:4801:20::/64",
        "2001:4860:4801:21::/64",
        "2001:4860:4801:22::/64",
        "2001:4860:4801:23::/64",
        "2001:4860:4801:24::/64",
        "2001:4860:4801:25::/64",
        "2001:4860:4801:26::/64",
        "2001:4860:4801:27::/64",
        "2001:4860:4801:28::/64",
        "2001:4860:4801:29::/64",
        "2001:4860:4801:2::/64",
        "2001:4860:4801:2a::/64",
        "2001:4860:4801:2b::/64",
        "2001:4860:4801:2c::/64",
        "2001:4860:4801:2d::/64",
        "2001:4860:4801:2e::/64",
        "2001:4860:4801:2f::/64",
        "2001:4860:4801:30::/64",
        "2001:4860:4801:31::/64",
        "2001:4860:4801:32::/64",
        "2001:4860:4801:33::/64",
        "2001:4860:4801:34::/64",
        "2001:4860:4801:35::/64",
        "2001:4860:4801:36::/64",
        "2001:4860:4801:37::/64",
        "2001:4860:4801:38::/64",
        "2001:4860:4801:39::/64",
        "2001:4860:4801:3a::/64",
        "2001:4860:4801:3b::/64",
        "2001:4860:4801:3c::/64",
        "2001:4860:4801:3d::/64",
        "2001:4860:4801:3e::/64",
        "2001:4860:4801:3f::/64",
        "2001:4860:4801:40::/64",
        "2001:4860:4801:41::/64",
        "2001:4860:4801:42::/64",
        "2001:4860:4801:44::/64",
        "2001:4860:4801:45::/64",
        "2001:4860:4801:46::/64",
        "2001:4860:4801:47::/64",
        "2001:4860:4801:48::/64",
        "2001:4860:4801:49::/64",
        "2001:4860:4801:4a::/64",
        "2001:4860:4801:4b::/64",
        "2001:4860:4801:4c::/64",
        "2001:4860:4801:4d::/64",
        "2001:4860:4801:4e::/64",
        "2001:4860:4801:50::/64",
        "2001:4860:4801:51::/64",
        "2001:4860:4801:52::/64",
        "2001:4860:4801:53::/64",
        "2001:4860:4801:54::/64",
        "2001:4860:4801:55::/64",
        "2001:4860:4801:56::/64",
        "2001:4860:4801:57::/64",
        "2001:4860:4801:58::/64",
        "2001:4860:4801:59::/64",
        "2001:4860:4801:60::/64",
        "2001:4860:4801:61::/64",
        "2001:4860:4801:62::/64",
        "2001:4860:4801:63::/64",
        "2001:4860:4801:64::/64",
        "2001:4860:4801:65::/64",
        "2001:4860:4801:66::/64",
        "2001:4860:4801:67::/64",
        "2001:4860:4801:68::/64",
        "2001:4860:4801:69::/64",
        "2001:4860:4801:6a::/64",
        "2001:4860:4801:6b::/64",
        "2001:4860:4801:6c::/64",
        "2001:4860:4801:6d::/64",
        "2001:4860:4801:6e::/64",
        "2001:4860:4801:6f::/64",
        "2001:4860:4801:70::/64",
        "2001:4860:4801:71::/64",
        "2001:4860:4801:72::/64",
        "2001:4860:4801:73::/64",
        "2001:4860:4801:74::/64",
        "2001:4860:4801:75::/64",
        "2001:4860:4801:76::/64",
        "2001:4860:4801:77::/64",
        "2001:4860:4801:78::/64",
        "2001:4860:4801:79::/64",
        "2001:4860:4801:7a::/64",
        "2001:4860:4801:7b::/64",
        "2001:4860:4801:7c::/64",
        "2001:4860:4801:7d::/64",
        "2001:4860:4801:7e::/64",
        "2001:4860:4801:7f::/64",
        "2001:4860:4801:80::/64",
        "2001:4860:4801:81::/64",
        "2001:4860:4801:82::/64",
        "2001:4860:4801:83::/64",
        "2001:4860:4801:84::/64",
        "2001:4860:4801:85::/64",
        "2001:4860:4801:86::/64",
        "2001:4860:4801:87::/64",
        "2001:4860:4801:88::/64",
        "2001:4860:4801:90::/64",
        "2001:4860:4801:91::/64",
        "2001:4860:4801:92::/64",
        "2001:4860:4801:93::/64",
        "2001:4860:4801:94::/64",
        "2001:4860:4801:95::/64",
        "2001:4860:4801:96::/64",
        "2001:4860:4801:97::/64",
        "2001:4860:4801:a0::/64",
        "2001:4860:4801:a1::/64",
        "2001:4860:4801:a2::/64",
        "2001:4860:4801:a3::/64",
        "2001:4860:4801:a4::/64",
        "2001:4860:4801:a5::/64",
        "2001:4860:4801:a6::/64",
        "2001:4860:4801:a7::/64",
        "2001:4860:4801:a8::/64",
        "2001:4860:4801:a9::/64",
        "2001:4860:4801:aa::/64",
        "2001:4860:4801:ab::/64",
        "2001:4860:4801:ac::/64",
        "2001:4860:4801:ad::/64",
        "2001:4860:4801:ae::/64",
        "2001:4860:4801:b0::/64",
        "2001:4860:4801:b1::/64",
        "2001:4860:4801:b2::/64",
        "2001:4860:4801:b3::/64",
        "2001:4860:4801:b4::/64",
        "2001:4860:4801:b5::/64",
        "2001:4860:4801:b6::/64",
        "2001:4860:4801:c::/64",
        "2001:4860:4801:f::/64",
        "34.100.182.96/28",
        "34.101.50.144/28",
        "34.118.254.0/28",
        "34.118.66.0/28",
        "34.126.178.96/28",
        "34.146.150.144/28",
        "34.147.110.144/28",
        "34.151.74.144/28",
        "34.152.50.64/28",
        "34.154.114.144/28",
        "34.155.98.32/28",
        "34.165.18.176/28",
        "34.175.160.64/28",
        "34.176.130.16/28",
        "34.22.85.0/27",
        "34.64.82.64/28",
        "34.65.242.112/28",
        "34.80.50.80/28",
        "34.88.194.0/28",
        "34.89.10.80/28",
        "34.89.198.80/28",
        "34.96.162.48/28",
        "35.247.243.240/28",
        "66.249.64.0/27",
        "66.249.64.128/27",
        "66.249.64.160/27",
        "66.249.64.192/27",
        "66.249.64.224/27",
        "66.249.64.32/27",
        "66.249.64.64/27",
        "66.249.64.96/27",
        "66.249.65.0/27",
        "66.249.65.128/27",
        "66.249.65.160/27",
        "66.249.65.192/27",
        "66.249.65.224/27",
        "66.249.65.32/27",
        "66.249.65.64/27",
        "66.249.65.96/27",
        "66.249.66.0/27",
        "66.249.66.128/27",
        "66.249.66.160/27",
        "66.249.66.192/27",
        "66.249.66.224/27",
        "66.249.66.32/27",
        "66.249.66.64/27",
        "66.249.66.96/27",
        "66.249.67.0/27",
        "66.249.67.32/27",
        "66.249.67.64/27",
        "66.249.68.0/27",
        "66.249.68.128/27",
        "66.249.68.160/27",
        "66.249.68.192/27",
        "66.249.68.32/27",
        "66.249.68.64/27",
        "66.249.68.96/27",
        "66.249.69.0/27",
        "66.249.69.128/27",
        "66.249.69.160/27",
        "66.249.69.192/27",
        "66.249.69.224/27",
        "66.249.69.32/27",
        "66.249.69.64/27",
        "66.249.69.96/27",
        "66.249.70.0/27",
        "66.249.70.128/27",
        "66.249.70.160/27",
        "66.249.70.192/27",
        "66.249.70.224/27",
        "66.249.70.32/27",
        "66.249.70.64/27",
        "66.249.70.96/27",
        "66.249.71.0/27",
        "66.249.71.128/27",
        "66.249.71.160/27",
        "66.249.71.192/27",
        "66.249.71.224/27",
        "66.249.71.32/27",
        "66.249.71.64/27",
        "66.249.71.96/27",
        "66.249.72.0/27",
        "66.249.72.128/27",
        "66.249.72.160/27",
        "66.249.72.192/27",
        "66.249.72.224/27",
        "66.249.72.32/27",
        "66.249.72.64/27",
        "66.249.72.96/27",
        "66.249.73.0/27",
        "66.249.73.128/27",
        "66.249.73.160/27",
        "66.249.73.192/27",
        "66.249.73.224/27",
        "66.249.73.32/27",
        "66.249.73.64/27",
        "66.249.73.96/27",
        "66.249.74.0/27",
        "66.249.74.128/27",
        "66.249.74.160/27",
        "66.249.74.192/27",
        "66.249.74.224/27",
        "66.249.74.32/27",
        "66.249.74.64/27",
        "66.249.74.96/27",
        "66.249.75.0/27",
        "66.249.75.128/27",
        "66.249.75.160/27",
        "66.249.75.192/27",
        "66.249.75.224/27",
        "66.249.75.32/27",
        "66.249.75.64/27",
        "66.249.75.96/27",
        "66.249.76.0/27",
        "66.249.76.128/27",
        "66.249.76.160/27",
        "66.249.76.192/27",
        "66.249.76.224/27",
        "66.249.76.32/27",
        "66.249.76.64/27",
        "66.249.76.96/27",
        "66.249.77.0/27",
        "66.249.77.128/27",
        "66.249.77.160/27",
        "66.249.77.192/27",
        "66.249.77.224/27",
        "66.249.77.32/27",
        "66.249.77.64/27",
        "66.249.77.96/27",
        "66.249.78.0/27",
        "66.249.78.128/27",
        "66.249.78.160/27",
        "66.249.78.192/27",
        "66.249.78.224/27",
        "66.249.78.32/27",
        "66.249.78.64/27",
        "66.249.78.96/27",
        "66.249.79.0/27",
        "66.249.79.128/27",
        "66.249.79.160/27",
        "66.249.79.192/27",
        "66.249.79.224/27",
        "66.249.79.32/27",
        "66.249.79.64/27"
      ]
    },
    {
      "id": "googlebot-video",
      "name": "Googlebot Video",
      "operator": {
        "name": "Google",
        "url": "https://www.google.com"
      },
      "description": "Google's video crawler, which fetches video content for Google Search video features. It is a distinct crawler from Googlebot with its own robots.txt token, though rules for Googlebot also apply to it.",
      "docs": [
        "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "Googlebot-Video/\\d"
        ],
        "instances": [
          "Googlebot-Video/1.0"
        ]
      },
      "behavior": {
        "respects_robots_txt": true,
        "robots_token": "Googlebot-Video"
      },
      "verification": [
        {
          "type": "cidr_feed",
          "url": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json",
          "format": "prefixes"
        },
        {
          "type": "rdns",
          "masks": [
            "*.googlebot.com",
            "*.geo.googlebot.com"
          ]
        }
      ],
      "tier": 1,
      "tier_label": "Fully verifiable",
      "first_seen": "2026-07-18T15:23:43.605Z",
      "resolved_cidrs": [
        "192.178.4.0/27",
        "192.178.4.128/27",
        "192.178.4.160/27",
        "192.178.4.192/27",
        "192.178.4.224/27",
        "192.178.4.32/27",
        "192.178.4.64/27",
        "192.178.4.96/27",
        "192.178.5.0/27",
        "192.178.6.0/27",
        "192.178.6.128/27",
        "192.178.6.160/27",
        "192.178.6.192/27",
        "192.178.6.224/27",
        "192.178.6.32/27",
        "192.178.6.64/27",
        "192.178.6.96/27",
        "192.178.7.0/27",
        "192.178.7.128/27",
        "192.178.7.160/27",
        "192.178.7.192/27",
        "192.178.7.224/27",
        "192.178.7.32/27",
        "192.178.7.64/27",
        "192.178.7.96/27",
        "2001:4860:4801:10::/64",
        "2001:4860:4801:11::/64",
        "2001:4860:4801:12::/64",
        "2001:4860:4801:13::/64",
        "2001:4860:4801:14::/64",
        "2001:4860:4801:15::/64",
        "2001:4860:4801:16::/64",
        "2001:4860:4801:17::/64",
        "2001:4860:4801:18::/64",
        "2001:4860:4801:19::/64",
        "2001:4860:4801:1a::/64",
        "2001:4860:4801:1b::/64",
        "2001:4860:4801:1c::/64",
        "2001:4860:4801:1d::/64",
        "2001:4860:4801:1e::/64",
        "2001:4860:4801:1f::/64",
        "2001:4860:4801:20::/64",
        "2001:4860:4801:21::/64",
        "2001:4860:4801:22::/64",
        "2001:4860:4801:23::/64",
        "2001:4860:4801:24::/64",
        "2001:4860:4801:25::/64",
        "2001:4860:4801:26::/64",
        "2001:4860:4801:27::/64",
        "2001:4860:4801:28::/64",
        "2001:4860:4801:29::/64",
        "2001:4860:4801:2::/64",
        "2001:4860:4801:2a::/64",
        "2001:4860:4801:2b::/64",
        "2001:4860:4801:2c::/64",
        "2001:4860:4801:2d::/64",
        "2001:4860:4801:2e::/64",
        "2001:4860:4801:2f::/64",
        "2001:4860:4801:30::/64",
        "2001:4860:4801:31::/64",
        "2001:4860:4801:32::/64",
        "2001:4860:4801:33::/64",
        "2001:4860:4801:34::/64",
        "2001:4860:4801:35::/64",
        "2001:4860:4801:36::/64",
        "2001:4860:4801:37::/64",
        "2001:4860:4801:38::/64",
        "2001:4860:4801:39::/64",
        "2001:4860:4801:3a::/64",
        "2001:4860:4801:3b::/64",
        "2001:4860:4801:3c::/64",
        "2001:4860:4801:3d::/64",
        "2001:4860:4801:3e::/64",
        "2001:4860:4801:3f::/64",
        "2001:4860:4801:40::/64",
        "2001:4860:4801:41::/64",
        "2001:4860:4801:42::/64",
        "2001:4860:4801:44::/64",
        "2001:4860:4801:45::/64",
        "2001:4860:4801:46::/64",
        "2001:4860:4801:47::/64",
        "2001:4860:4801:48::/64",
        "2001:4860:4801:49::/64",
        "2001:4860:4801:4a::/64",
        "2001:4860:4801:4b::/64",
        "2001:4860:4801:4c::/64",
        "2001:4860:4801:4d::/64",
        "2001:4860:4801:4e::/64",
        "2001:4860:4801:50::/64",
        "2001:4860:4801:51::/64",
        "2001:4860:4801:52::/64",
        "2001:4860:4801:53::/64",
        "2001:4860:4801:54::/64",
        "2001:4860:4801:55::/64",
        "2001:4860:4801:56::/64",
        "2001:4860:4801:57::/64",
        "2001:4860:4801:58::/64",
        "2001:4860:4801:59::/64",
        "2001:4860:4801:60::/64",
        "2001:4860:4801:61::/64",
        "2001:4860:4801:62::/64",
        "2001:4860:4801:63::/64",
        "2001:4860:4801:64::/64",
        "2001:4860:4801:65::/64",
        "2001:4860:4801:66::/64",
        "2001:4860:4801:67::/64",
        "2001:4860:4801:68::/64",
        "2001:4860:4801:69::/64",
        "2001:4860:4801:6a::/64",
        "2001:4860:4801:6b::/64",
        "2001:4860:4801:6c::/64",
        "2001:4860:4801:6d::/64",
        "2001:4860:4801:6e::/64",
        "2001:4860:4801:6f::/64",
        "2001:4860:4801:70::/64",
        "2001:4860:4801:71::/64",
        "2001:4860:4801:72::/64",
        "2001:4860:4801:73::/64",
        "2001:4860:4801:74::/64",
        "2001:4860:4801:75::/64",
        "2001:4860:4801:76::/64",
        "2001:4860:4801:77::/64",
        "2001:4860:4801:78::/64",
        "2001:4860:4801:79::/64",
        "2001:4860:4801:7a::/64",
        "2001:4860:4801:7b::/64",
        "2001:4860:4801:7c::/64",
        "2001:4860:4801:7d::/64",
        "2001:4860:4801:7e::/64",
        "2001:4860:4801:7f::/64",
        "2001:4860:4801:80::/64",
        "2001:4860:4801:81::/64",
        "2001:4860:4801:82::/64",
        "2001:4860:4801:83::/64",
        "2001:4860:4801:84::/64",
        "2001:4860:4801:85::/64",
        "2001:4860:4801:86::/64",
        "2001:4860:4801:87::/64",
        "2001:4860:4801:88::/64",
        "2001:4860:4801:90::/64",
        "2001:4860:4801:91::/64",
        "2001:4860:4801:92::/64",
        "2001:4860:4801:93::/64",
        "2001:4860:4801:94::/64",
        "2001:4860:4801:95::/64",
        "2001:4860:4801:96::/64",
        "2001:4860:4801:97::/64",
        "2001:4860:4801:a0::/64",
        "2001:4860:4801:a1::/64",
        "2001:4860:4801:a2::/64",
        "2001:4860:4801:a3::/64",
        "2001:4860:4801:a4::/64",
        "2001:4860:4801:a5::/64",
        "2001:4860:4801:a6::/64",
        "2001:4860:4801:a7::/64",
        "2001:4860:4801:a8::/64",
        "2001:4860:4801:a9::/64",
        "2001:4860:4801:aa::/64",
        "2001:4860:4801:ab::/64",
        "2001:4860:4801:ac::/64",
        "2001:4860:4801:ad::/64",
        "2001:4860:4801:ae::/64",
        "2001:4860:4801:b0::/64",
        "2001:4860:4801:b1::/64",
        "2001:4860:4801:b2::/64",
        "2001:4860:4801:b3::/64",
        "2001:4860:4801:b4::/64",
        "2001:4860:4801:b5::/64",
        "2001:4860:4801:b6::/64",
        "2001:4860:4801:c::/64",
        "2001:4860:4801:f::/64",
        "34.100.182.96/28",
        "34.101.50.144/28",
        "34.118.254.0/28",
        "34.118.66.0/28",
        "34.126.178.96/28",
        "34.146.150.144/28",
        "34.147.110.144/28",
        "34.151.74.144/28",
        "34.152.50.64/28",
        "34.154.114.144/28",
        "34.155.98.32/28",
        "34.165.18.176/28",
        "34.175.160.64/28",
        "34.176.130.16/28",
        "34.22.85.0/27",
        "34.64.82.64/28",
        "34.65.242.112/28",
        "34.80.50.80/28",
        "34.88.194.0/28",
        "34.89.10.80/28",
        "34.89.198.80/28",
        "34.96.162.48/28",
        "35.247.243.240/28",
        "66.249.64.0/27",
        "66.249.64.128/27",
        "66.249.64.160/27",
        "66.249.64.192/27",
        "66.249.64.224/27",
        "66.249.64.32/27",
        "66.249.64.64/27",
        "66.249.64.96/27",
        "66.249.65.0/27",
        "66.249.65.128/27",
        "66.249.65.160/27",
        "66.249.65.192/27",
        "66.249.65.224/27",
        "66.249.65.32/27",
        "66.249.65.64/27",
        "66.249.65.96/27",
        "66.249.66.0/27",
        "66.249.66.128/27",
        "66.249.66.160/27",
        "66.249.66.192/27",
        "66.249.66.224/27",
        "66.249.66.32/27",
        "66.249.66.64/27",
        "66.249.66.96/27",
        "66.249.67.0/27",
        "66.249.67.32/27",
        "66.249.67.64/27",
        "66.249.68.0/27",
        "66.249.68.128/27",
        "66.249.68.160/27",
        "66.249.68.192/27",
        "66.249.68.32/27",
        "66.249.68.64/27",
        "66.249.68.96/27",
        "66.249.69.0/27",
        "66.249.69.128/27",
        "66.249.69.160/27",
        "66.249.69.192/27",
        "66.249.69.224/27",
        "66.249.69.32/27",
        "66.249.69.64/27",
        "66.249.69.96/27",
        "66.249.70.0/27",
        "66.249.70.128/27",
        "66.249.70.160/27",
        "66.249.70.192/27",
        "66.249.70.224/27",
        "66.249.70.32/27",
        "66.249.70.64/27",
        "66.249.70.96/27",
        "66.249.71.0/27",
        "66.249.71.128/27",
        "66.249.71.160/27",
        "66.249.71.192/27",
        "66.249.71.224/27",
        "66.249.71.32/27",
        "66.249.71.64/27",
        "66.249.71.96/27",
        "66.249.72.0/27",
        "66.249.72.128/27",
        "66.249.72.160/27",
        "66.249.72.192/27",
        "66.249.72.224/27",
        "66.249.72.32/27",
        "66.249.72.64/27",
        "66.249.72.96/27",
        "66.249.73.0/27",
        "66.249.73.128/27",
        "66.249.73.160/27",
        "66.249.73.192/27",
        "66.249.73.224/27",
        "66.249.73.32/27",
        "66.249.73.64/27",
        "66.249.73.96/27",
        "66.249.74.0/27",
        "66.249.74.128/27",
        "66.249.74.160/27",
        "66.249.74.192/27",
        "66.249.74.224/27",
        "66.249.74.32/27",
        "66.249.74.64/27",
        "66.249.74.96/27",
        "66.249.75.0/27",
        "66.249.75.128/27",
        "66.249.75.160/27",
        "66.249.75.192/27",
        "66.249.75.224/27",
        "66.249.75.32/27",
        "66.249.75.64/27",
        "66.249.75.96/27",
        "66.249.76.0/27",
        "66.249.76.128/27",
        "66.249.76.160/27",
        "66.249.76.192/27",
        "66.249.76.224/27",
        "66.249.76.32/27",
        "66.249.76.64/27",
        "66.249.76.96/27",
        "66.249.77.0/27",
        "66.249.77.128/27",
        "66.249.77.160/27",
        "66.249.77.192/27",
        "66.249.77.224/27",
        "66.249.77.32/27",
        "66.249.77.64/27",
        "66.249.77.96/27",
        "66.249.78.0/27",
        "66.249.78.128/27",
        "66.249.78.160/27",
        "66.249.78.192/27",
        "66.249.78.224/27",
        "66.249.78.32/27",
        "66.249.78.64/27",
        "66.249.78.96/27",
        "66.249.79.0/27",
        "66.249.79.128/27",
        "66.249.79.160/27",
        "66.249.79.192/27",
        "66.249.79.224/27",
        "66.249.79.32/27",
        "66.249.79.64/27"
      ]
    },
    {
      "id": "googleother",
      "name": "GoogleOther",
      "operator": {
        "name": "Google",
        "url": "https://www.google.com"
      },
      "description": "Google's generic crawler family (GoogleOther, GoogleOther-Image, GoogleOther-Video), used by various Google product teams for fetching publicly accessible content. Robots.txt rules addressed to it do not affect Google Search.",
      "docs": [
        "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "GoogleOther"
        ],
        "instances": [
          "GoogleOther",
          "GoogleOther-Image/1.0",
          "GoogleOther-Video/1.0"
        ]
      },
      "behavior": {
        "respects_robots_txt": true,
        "robots_token": "GoogleOther"
      },
      "verification": [
        {
          "type": "cidr_feed",
          "url": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json",
          "format": "prefixes"
        }
      ],
      "tier": 1,
      "tier_label": "Fully verifiable",
      "first_seen": "2026-07-09T03:01:47.690Z",
      "resolved_cidrs": [
        "192.178.4.0/27",
        "192.178.4.128/27",
        "192.178.4.160/27",
        "192.178.4.192/27",
        "192.178.4.224/27",
        "192.178.4.32/27",
        "192.178.4.64/27",
        "192.178.4.96/27",
        "192.178.5.0/27",
        "192.178.6.0/27",
        "192.178.6.128/27",
        "192.178.6.160/27",
        "192.178.6.192/27",
        "192.178.6.224/27",
        "192.178.6.32/27",
        "192.178.6.64/27",
        "192.178.6.96/27",
        "192.178.7.0/27",
        "192.178.7.128/27",
        "192.178.7.160/27",
        "192.178.7.192/27",
        "192.178.7.224/27",
        "192.178.7.32/27",
        "192.178.7.64/27",
        "192.178.7.96/27",
        "2001:4860:4801:10::/64",
        "2001:4860:4801:11::/64",
        "2001:4860:4801:12::/64",
        "2001:4860:4801:13::/64",
        "2001:4860:4801:14::/64",
        "2001:4860:4801:15::/64",
        "2001:4860:4801:16::/64",
        "2001:4860:4801:17::/64",
        "2001:4860:4801:18::/64",
        "2001:4860:4801:19::/64",
        "2001:4860:4801:1a::/64",
        "2001:4860:4801:1b::/64",
        "2001:4860:4801:1c::/64",
        "2001:4860:4801:1d::/64",
        "2001:4860:4801:1e::/64",
        "2001:4860:4801:1f::/64",
        "2001:4860:4801:20::/64",
        "2001:4860:4801:21::/64",
        "2001:4860:4801:22::/64",
        "2001:4860:4801:23::/64",
        "2001:4860:4801:24::/64",
        "2001:4860:4801:25::/64",
        "2001:4860:4801:26::/64",
        "2001:4860:4801:27::/64",
        "2001:4860:4801:28::/64",
        "2001:4860:4801:29::/64",
        "2001:4860:4801:2::/64",
        "2001:4860:4801:2a::/64",
        "2001:4860:4801:2b::/64",
        "2001:4860:4801:2c::/64",
        "2001:4860:4801:2d::/64",
        "2001:4860:4801:2e::/64",
        "2001:4860:4801:2f::/64",
        "2001:4860:4801:30::/64",
        "2001:4860:4801:31::/64",
        "2001:4860:4801:32::/64",
        "2001:4860:4801:33::/64",
        "2001:4860:4801:34::/64",
        "2001:4860:4801:35::/64",
        "2001:4860:4801:36::/64",
        "2001:4860:4801:37::/64",
        "2001:4860:4801:38::/64",
        "2001:4860:4801:39::/64",
        "2001:4860:4801:3a::/64",
        "2001:4860:4801:3b::/64",
        "2001:4860:4801:3c::/64",
        "2001:4860:4801:3d::/64",
        "2001:4860:4801:3e::/64",
        "2001:4860:4801:3f::/64",
        "2001:4860:4801:40::/64",
        "2001:4860:4801:41::/64",
        "2001:4860:4801:42::/64",
        "2001:4860:4801:44::/64",
        "2001:4860:4801:45::/64",
        "2001:4860:4801:46::/64",
        "2001:4860:4801:47::/64",
        "2001:4860:4801:48::/64",
        "2001:4860:4801:49::/64",
        "2001:4860:4801:4a::/64",
        "2001:4860:4801:4b::/64",
        "2001:4860:4801:4c::/64",
        "2001:4860:4801:4d::/64",
        "2001:4860:4801:4e::/64",
        "2001:4860:4801:50::/64",
        "2001:4860:4801:51::/64",
        "2001:4860:4801:52::/64",
        "2001:4860:4801:53::/64",
        "2001:4860:4801:54::/64",
        "2001:4860:4801:55::/64",
        "2001:4860:4801:56::/64",
        "2001:4860:4801:57::/64",
        "2001:4860:4801:58::/64",
        "2001:4860:4801:59::/64",
        "2001:4860:4801:60::/64",
        "2001:4860:4801:61::/64",
        "2001:4860:4801:62::/64",
        "2001:4860:4801:63::/64",
        "2001:4860:4801:64::/64",
        "2001:4860:4801:65::/64",
        "2001:4860:4801:66::/64",
        "2001:4860:4801:67::/64",
        "2001:4860:4801:68::/64",
        "2001:4860:4801:69::/64",
        "2001:4860:4801:6a::/64",
        "2001:4860:4801:6b::/64",
        "2001:4860:4801:6c::/64",
        "2001:4860:4801:6d::/64",
        "2001:4860:4801:6e::/64",
        "2001:4860:4801:6f::/64",
        "2001:4860:4801:70::/64",
        "2001:4860:4801:71::/64",
        "2001:4860:4801:72::/64",
        "2001:4860:4801:73::/64",
        "2001:4860:4801:74::/64",
        "2001:4860:4801:75::/64",
        "2001:4860:4801:76::/64",
        "2001:4860:4801:77::/64",
        "2001:4860:4801:78::/64",
        "2001:4860:4801:79::/64",
        "2001:4860:4801:7a::/64",
        "2001:4860:4801:7b::/64",
        "2001:4860:4801:7c::/64",
        "2001:4860:4801:7d::/64",
        "2001:4860:4801:7e::/64",
        "2001:4860:4801:7f::/64",
        "2001:4860:4801:80::/64",
        "2001:4860:4801:81::/64",
        "2001:4860:4801:82::/64",
        "2001:4860:4801:83::/64",
        "2001:4860:4801:84::/64",
        "2001:4860:4801:85::/64",
        "2001:4860:4801:86::/64",
        "2001:4860:4801:87::/64",
        "2001:4860:4801:88::/64",
        "2001:4860:4801:90::/64",
        "2001:4860:4801:91::/64",
        "2001:4860:4801:92::/64",
        "2001:4860:4801:93::/64",
        "2001:4860:4801:94::/64",
        "2001:4860:4801:95::/64",
        "2001:4860:4801:96::/64",
        "2001:4860:4801:97::/64",
        "2001:4860:4801:a0::/64",
        "2001:4860:4801:a1::/64",
        "2001:4860:4801:a2::/64",
        "2001:4860:4801:a3::/64",
        "2001:4860:4801:a4::/64",
        "2001:4860:4801:a5::/64",
        "2001:4860:4801:a6::/64",
        "2001:4860:4801:a7::/64",
        "2001:4860:4801:a8::/64",
        "2001:4860:4801:a9::/64",
        "2001:4860:4801:aa::/64",
        "2001:4860:4801:ab::/64",
        "2001:4860:4801:ac::/64",
        "2001:4860:4801:ad::/64",
        "2001:4860:4801:ae::/64",
        "2001:4860:4801:b0::/64",
        "2001:4860:4801:b1::/64",
        "2001:4860:4801:b2::/64",
        "2001:4860:4801:b3::/64",
        "2001:4860:4801:b4::/64",
        "2001:4860:4801:b5::/64",
        "2001:4860:4801:b6::/64",
        "2001:4860:4801:c::/64",
        "2001:4860:4801:f::/64",
        "34.100.182.96/28",
        "34.101.50.144/28",
        "34.118.254.0/28",
        "34.118.66.0/28",
        "34.126.178.96/28",
        "34.146.150.144/28",
        "34.147.110.144/28",
        "34.151.74.144/28",
        "34.152.50.64/28",
        "34.154.114.144/28",
        "34.155.98.32/28",
        "34.165.18.176/28",
        "34.175.160.64/28",
        "34.176.130.16/28",
        "34.22.85.0/27",
        "34.64.82.64/28",
        "34.65.242.112/28",
        "34.80.50.80/28",
        "34.88.194.0/28",
        "34.89.10.80/28",
        "34.89.198.80/28",
        "34.96.162.48/28",
        "35.247.243.240/28",
        "66.249.64.0/27",
        "66.249.64.128/27",
        "66.249.64.160/27",
        "66.249.64.192/27",
        "66.249.64.224/27",
        "66.249.64.32/27",
        "66.249.64.64/27",
        "66.249.64.96/27",
        "66.249.65.0/27",
        "66.249.65.128/27",
        "66.249.65.160/27",
        "66.249.65.192/27",
        "66.249.65.224/27",
        "66.249.65.32/27",
        "66.249.65.64/27",
        "66.249.65.96/27",
        "66.249.66.0/27",
        "66.249.66.128/27",
        "66.249.66.160/27",
        "66.249.66.192/27",
        "66.249.66.224/27",
        "66.249.66.32/27",
        "66.249.66.64/27",
        "66.249.66.96/27",
        "66.249.67.0/27",
        "66.249.67.32/27",
        "66.249.67.64/27",
        "66.249.68.0/27",
        "66.249.68.128/27",
        "66.249.68.160/27",
        "66.249.68.192/27",
        "66.249.68.32/27",
        "66.249.68.64/27",
        "66.249.68.96/27",
        "66.249.69.0/27",
        "66.249.69.128/27",
        "66.249.69.160/27",
        "66.249.69.192/27",
        "66.249.69.224/27",
        "66.249.69.32/27",
        "66.249.69.64/27",
        "66.249.69.96/27",
        "66.249.70.0/27",
        "66.249.70.128/27",
        "66.249.70.160/27",
        "66.249.70.192/27",
        "66.249.70.224/27",
        "66.249.70.32/27",
        "66.249.70.64/27",
        "66.249.70.96/27",
        "66.249.71.0/27",
        "66.249.71.128/27",
        "66.249.71.160/27",
        "66.249.71.192/27",
        "66.249.71.224/27",
        "66.249.71.32/27",
        "66.249.71.64/27",
        "66.249.71.96/27",
        "66.249.72.0/27",
        "66.249.72.128/27",
        "66.249.72.160/27",
        "66.249.72.192/27",
        "66.249.72.224/27",
        "66.249.72.32/27",
        "66.249.72.64/27",
        "66.249.72.96/27",
        "66.249.73.0/27",
        "66.249.73.128/27",
        "66.249.73.160/27",
        "66.249.73.192/27",
        "66.249.73.224/27",
        "66.249.73.32/27",
        "66.249.73.64/27",
        "66.249.73.96/27",
        "66.249.74.0/27",
        "66.249.74.128/27",
        "66.249.74.160/27",
        "66.249.74.192/27",
        "66.249.74.224/27",
        "66.249.74.32/27",
        "66.249.74.64/27",
        "66.249.74.96/27",
        "66.249.75.0/27",
        "66.249.75.128/27",
        "66.249.75.160/27",
        "66.249.75.192/27",
        "66.249.75.224/27",
        "66.249.75.32/27",
        "66.249.75.64/27",
        "66.249.75.96/27",
        "66.249.76.0/27",
        "66.249.76.128/27",
        "66.249.76.160/27",
        "66.249.76.192/27",
        "66.249.76.224/27",
        "66.249.76.32/27",
        "66.249.76.64/27",
        "66.249.76.96/27",
        "66.249.77.0/27",
        "66.249.77.128/27",
        "66.249.77.160/27",
        "66.249.77.192/27",
        "66.249.77.224/27",
        "66.249.77.32/27",
        "66.249.77.64/27",
        "66.249.77.96/27",
        "66.249.78.0/27",
        "66.249.78.128/27",
        "66.249.78.160/27",
        "66.249.78.192/27",
        "66.249.78.224/27",
        "66.249.78.32/27",
        "66.249.78.64/27",
        "66.249.78.96/27",
        "66.249.79.0/27",
        "66.249.79.128/27",
        "66.249.79.160/27",
        "66.249.79.192/27",
        "66.249.79.224/27",
        "66.249.79.32/27",
        "66.249.79.64/27"
      ]
    },
    {
      "id": "haosou-crawler",
      "name": "360Spider",
      "operator": {
        "name": "360 Search (Qihoo 360)",
        "url": "https://www.so.com/"
      },
      "description": "360Spider is the web-search crawler of 360 Search (so.com), the search engine operated by Chinese internet company Qihoo 360. The operator also runs 360Spider-Image and 360Spider-Video for image and video search.",
      "docs": [
        "https://www.so.com/help/help_3_2.html"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "360Spider"
        ],
        "instances": [
          "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1; 360Spider"
        ]
      },
      "behavior": {
        "respects_robots_txt": true,
        "robots_token": "360Spider"
      },
      "verification": [],
      "tier": 3,
      "tier_label": "Listed only",
      "first_seen": "2026-07-16T00:59:46.504Z",
      "resolved_cidrs": []
    },
    {
      "id": "ioncrawl",
      "name": "IONOS Crawler",
      "operator": {
        "name": "IONOS",
        "url": "https://www.ionos.com"
      },
      "description": "IONOS Crawler (IonCrawl) continuously crawls publicly accessible domains to generate insights into how they are used, which IONOS uses to improve and expand its hosting products. IONOS documents reverse-DNS verification and deletes crawled data after 60 days.",
      "docs": [
        "https://www.ionos.de/terms-gtc/faq-crawler-en"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "IonCrawl"
        ],
        "instances": [
          "IonCrawl (https://www.ionos.de/terms-gtc/faq-crawler-en/)"
        ]
      },
      "behavior": {
        "respects_robots_txt": true,
        "robots_token": "IonCrawl"
      },
      "verification": [
        {
          "type": "rdns",
          "masks": [
            "*.1and1.org"
          ]
        }
      ],
      "tier": 2,
      "tier_label": "Verifiable",
      "first_seen": "2026-07-24T13:19:52.933Z",
      "resolved_cidrs": []
    },
    {
      "id": "jooblebot",
      "name": "Jooblebot",
      "operator": {
        "name": "Jooble",
        "url": "https://jooble.org"
      },
      "description": "The crawler for Jooble, a job-search aggregator that indexes job listings published across the web. Jooble's bot page states that it uses a web crawler which identifies itself as JoobleBot. No IP ranges are published.",
      "docs": [
        "https://jooble.org/jooble-bot"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "Jooblebot/[\\d.]+"
        ],
        "instances": [
          "Mozilla/5.0 (compatible; Jooblebot/2.0; Windows NT 6.1; WOW64; +http://jooble.org/jooble-bot) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36"
        ]
      },
      "behavior": {
        "respects_robots_txt": false
      },
      "verification": [],
      "tier": 3,
      "tier_label": "Listed only",
      "first_seen": "2026-07-30T16:13:28.285Z",
      "resolved_cidrs": []
    },
    {
      "id": "kagibot",
      "name": "Kagibot",
      "operator": {
        "name": "Kagi",
        "url": "https://kagi.com"
      },
      "description": "Kagi's web crawler, fetching pages for the paid Kagi search engine. Kagi publishes a fixed, small set of source IPs rather than a CIDR feed, each paired with a kagibot.org hostname.",
      "docs": [
        "https://kagi.com/bot"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "Kagibot/\\d"
        ],
        "instances": [
          "Mozilla/5.0 (compatible; Kagibot/1.0; +https://kagi.com/bot)"
        ]
      },
      "behavior": {
        "respects_robots_txt": true,
        "robots_token": "Kagibot"
      },
      "verification": [
        {
          "type": "static_cidrs",
          "cidrs": [
            "216.18.205.234/32",
            "35.212.27.76/32",
            "104.254.65.50/32",
            "209.151.156.194/32"
          ]
        },
        {
          "type": "rdns",
          "masks": [
            "*.kagibot.org"
          ]
        }
      ],
      "tier": 2,
      "tier_label": "Verifiable",
      "first_seen": "2026-07-08T19:13:49.571Z",
      "resolved_cidrs": [
        "104.254.65.50/32",
        "209.151.156.194/32",
        "216.18.205.234/32",
        "35.212.27.76/32"
      ]
    },
    {
      "id": "linespider",
      "name": "Linespider",
      "operator": {
        "name": "LINE",
        "url": "https://line.me"
      },
      "description": "LINE's crawler. It fetches web pages to support LINE's search features and content services, and adheres to the Robots Exclusion Protocol outlined in robots.txt.",
      "docs": [
        "https://help2.line.me/linesearchbot/web/?contentId=50006055&lang=en"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "Linespider"
        ],
        "instances": [
          "Mozilla/5.0 (compatible; Linespider/1.1; +https://lin.ee/4dwXkTH)",
          "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Linespider/1.1; +https://lin.ee/4dwXkTH) Chrome/W.X.Y.Z Safari/537.36"
        ]
      },
      "behavior": {
        "respects_robots_txt": true,
        "robots_token": "Linespider"
      },
      "verification": [],
      "tier": 3,
      "tier_label": "Listed only",
      "first_seen": "2026-07-16T13:18:54.422Z",
      "resolved_cidrs": []
    },
    {
      "id": "marginalia",
      "name": "Marginalia",
      "operator": {
        "name": "Marginalia Search",
        "url": "https://www.marginalia.nu"
      },
      "description": "The crawler behind Marginalia Search, an independent search engine focused on older, text-heavy, and non-commercial web pages. Identifies itself by a bare URL rather than a conventional bot token.",
      "docs": [
        "https://about.marginalia-search.com/article/crawler/"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "search\\.marginalia\\.nu"
        ],
        "instances": [
          "search.marginalia.nu"
        ]
      },
      "behavior": {
        "respects_robots_txt": true,
        "robots_token": "search.marginalia.nu"
      },
      "verification": [
        {
          "type": "cidr_feed",
          "url": "https://search.marginalia.nu/crawler-ips.txt",
          "format": "text_lines"
        }
      ],
      "tier": 1,
      "tier_label": "Fully verifiable",
      "first_seen": "2026-07-08T19:13:49.571Z",
      "resolved_cidrs": [
        "193.183.0.162/32",
        "193.183.0.163/32",
        "193.183.0.164/32",
        "193.183.0.165/32",
        "193.183.0.166/32",
        "193.183.0.167/32",
        "193.183.0.168/32",
        "193.183.0.169/32",
        "193.183.0.170/32",
        "193.183.0.171/32",
        "193.183.0.172/32",
        "193.183.0.173/32",
        "193.183.0.174/32",
        "81.170.128.52/32"
      ]
    },
    {
      "id": "metajob-crawler",
      "name": "MetaJobBot",
      "operator": {
        "name": "METAJob",
        "url": "https://www.metajob.de"
      },
      "description": "MetaJobBot is the focused web crawler operated by the METAJob job meta-search engine. It searches websites for job listings to include in the METAJob index.",
      "docs": [
        "https://www.metajob.de/the/crawler"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "MetaJobBot"
        ],
        "instances": [
          "Mozilla/5.0 (compatible; MetaJobBot; http://www.metajob.de/crawler)"
        ]
      },
      "behavior": {
        "respects_robots_txt": true,
        "robots_token": "metajobbot"
      },
      "verification": [
        {
          "type": "static_cidrs",
          "cidrs": [
            "157.90.179.220/32",
            "157.90.179.228/32",
            "157.90.179.241/32"
          ]
        }
      ],
      "tier": 2,
      "tier_label": "Verifiable",
      "first_seen": "2026-07-16T01:14:20.999Z",
      "resolved_cidrs": [
        "157.90.179.220/32",
        "157.90.179.228/32",
        "157.90.179.241/32"
      ]
    },
    {
      "id": "mojeekbot",
      "name": "MojeekBot",
      "operator": {
        "name": "Mojeek",
        "url": "https://www.mojeek.com"
      },
      "description": "Mojeek's web crawler, fetching pages for the independent Mojeek search engine index.",
      "docs": [
        "https://www.mojeek.com/bot.html"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "MojeekBot/\\d"
        ],
        "instances": [
          "Mozilla/5.0 (compatible; MojeekBot/0.6; +https://www.mojeek.com/bot.html)",
          "MojeekBot/0.2 (archi; http://www.mojeek.com/bot.html)"
        ]
      },
      "behavior": {
        "respects_robots_txt": true,
        "robots_token": "MojeekBot"
      },
      "verification": [
        {
          "type": "cidr_feed",
          "url": "https://www.mojeek.com/mojeekbot.json",
          "format": "prefixes"
        }
      ],
      "tier": 1,
      "tier_label": "Fully verifiable",
      "first_seen": "2026-07-08T19:13:49.571Z",
      "resolved_cidrs": [
        "5.102.173.64/28"
      ]
    },
    {
      "id": "naver-yeti",
      "name": "Yeti",
      "operator": {
        "name": "Naver",
        "url": "https://www.naver.com"
      },
      "description": "Naver's web crawler, fetching pages for Naver Search indexing across South Korea's largest search engine.",
      "docs": [
        "https://searchadvisor.naver.com/guide/seo-basic-firewall"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "Yeti/\\d"
        ],
        "instances": [
          "Mozilla/5.0 (compatible; Yeti/1.1; +http://naver.me/spd)",
          "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Yeti/1.1; +https://naver.me/spd) Chrome/106.0.5249.0 Safari/537.36"
        ]
      },
      "behavior": {
        "respects_robots_txt": true,
        "robots_token": "Yeti"
      },
      "verification": [
        {
          "type": "rdns",
          "masks": [
            "*.naver.com"
          ]
        }
      ],
      "tier": 2,
      "tier_label": "Verifiable",
      "first_seen": "2026-07-08T19:13:49.571Z",
      "resolved_cidrs": []
    },
    {
      "id": "openindex-crawler",
      "name": "Openindex Spider",
      "operator": {
        "name": "Openindex",
        "url": "https://www.openindex.io"
      },
      "description": "Openindex operates a web-crawling cluster (Apache Nutch on an Apache Hadoop cluster) for research and development of universal and focused search engines. The crawler respects robots.txt and the Crawl-delay directive and identifies itself in the User-Agent.",
      "docs": [
        "https://www.openindex.io/about-our-spider/"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "OpenindexSpider"
        ],
        "instances": [
          "Mozilla/5.0 (compatible; OpenindexSpider; +https://www.openindex.io/saas/about-our-spider/)"
        ]
      },
      "behavior": {
        "respects_robots_txt": true,
        "robots_token": "OpenindexSpider"
      },
      "verification": [],
      "tier": 3,
      "tier_label": "Listed only",
      "first_seen": "2026-07-16T00:59:46.504Z",
      "resolved_cidrs": []
    },
    {
      "id": "petalbot",
      "name": "PetalBot",
      "operator": {
        "name": "Huawei",
        "url": "https://www.huawei.com"
      },
      "description": "Huawei's web crawler, fetching pages for the Petal Search engine used in Huawei's mobile services and for Huawei Assistant content recommendations. Huawei documents reverse-DNS verification against aspiegel.com and petalsearch.com hostnames.",
      "docs": [
        "https://aspiegel.com/petalbot"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "PetalBot"
        ],
        "instances": [
          "Mozilla/5.0 (compatible;PetalBot;+https://webmaster.petalsearch.com/site/petalbot)",
          "Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot;+https://webmaster.petalsearch.com/site/petalbot)"
        ]
      },
      "behavior": {
        "respects_robots_txt": true,
        "robots_token": "PetalBot"
      },
      "verification": [
        {
          "type": "rdns",
          "masks": [
            "*.petalsearch.com",
            "*.aspiegel.com"
          ]
        }
      ],
      "tier": 2,
      "tier_label": "Verifiable",
      "first_seen": "2026-07-08T19:13:49.571Z",
      "resolved_cidrs": []
    },
    {
      "id": "pipl-crawler",
      "name": "PiplBot",
      "operator": {
        "name": "Pipl",
        "url": "https://pipl.com"
      },
      "description": "PiplBot is the web-indexing crawler operated by Pipl, an identity and people-search service. It retrieves and indexes publicly available information, including content behind searchable databases, to build Pipl's people-search index.",
      "docs": [
        "https://pipl.com/bot/"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "PiplBot"
        ],
        "instances": [
          "PiplBot (+http://www.pipl.com/bot/)",
          "Mozilla/5.0+(compatible;+PiplBot;+http://www.pipl.com/bot/)"
        ]
      },
      "behavior": {
        "respects_robots_txt": true,
        "robots_token": "PiplBot"
      },
      "verification": [],
      "tier": 3,
      "tier_label": "Listed only",
      "first_seen": "2026-07-16T01:14:20.999Z",
      "resolved_cidrs": []
    },
    {
      "id": "quantcast-crawler",
      "name": "Quantcastbot",
      "operator": {
        "name": "Quantcast",
        "url": "https://www.quantcast.com"
      },
      "description": "Quantcast's advertising crawler. It crawls websites to extract page content for interest-based audience categorization and to run quality-assurance checks on advertisement landing pages.",
      "docs": [
        "https://www.quantcast.com/bot"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "Quantcastbot/\\d"
        ],
        "instances": [
          "Quantcastbot/1.0 (+https://www.quantcast.com/bot)",
          "Quantcastbot/2.0 (+https://www.quantcast.com/bot)"
        ]
      },
      "behavior": {
        "respects_robots_txt": true
      },
      "verification": [
        {
          "type": "static_cidrs",
          "cidrs": [
            "52.34.88.131/32",
            "54.71.154.151/32",
            "192.184.74.253/32",
            "192.184.75.253/32",
            "192.184.76.253/32",
            "192.184.77.253/32",
            "103.229.11.253/32"
          ]
        }
      ],
      "tier": 2,
      "tier_label": "Verifiable",
      "first_seen": "2026-07-16T00:59:46.504Z",
      "resolved_cidrs": [
        "103.229.11.253/32",
        "192.184.74.253/32",
        "192.184.75.253/32",
        "192.184.76.253/32",
        "192.184.77.253/32",
        "52.34.88.131/32",
        "54.71.154.151/32"
      ]
    },
    {
      "id": "qwantbot",
      "name": "Qwantbot",
      "operator": {
        "name": "Qwant",
        "url": "https://www.qwant.com"
      },
      "description": "Qwant's web crawler, fetching pages for the privacy-focused Qwant search engine. Older crawls identify as Qwantify; current documentation uses the Qwantbot name.",
      "docs": [
        "https://help.qwant.com/bot/"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "Qwantify/\\d",
          "Qwantbot/\\d"
        ],
        "instances": [
          "Mozilla/5.0 (compatible; Qwantify/2.4w; +https://www.qwant.com/)/2.4w",
          "Mozilla/5.0 (compatible; Qwantbot/1.0_12345; +https://help.qwant.com/bot/)"
        ]
      },
      "behavior": {
        "respects_robots_txt": true
      },
      "verification": [
        {
          "type": "cidr_feed",
          "url": "https://help.qwant.com/wp-content/uploads/sites/latest/qwantbot.json",
          "format": "prefixes"
        },
        {
          "type": "rdns",
          "masks": [
            "*.qwant.com"
          ]
        }
      ],
      "tier": 1,
      "tier_label": "Fully verifiable",
      "first_seen": "2026-07-08T19:13:49.571Z",
      "resolved_cidrs": [
        "194.187.171.0/24"
      ]
    },
    {
      "id": "seekport",
      "name": "SeekportBot",
      "operator": {
        "name": "SISTRIX",
        "url": "https://www.sistrix.com"
      },
      "description": "The crawler for the Seekport search engine, operated by SISTRIX (Bonn, Germany). It fetches pages to build Seekport's search index and follows the Disallow directives in robots.txt.",
      "docs": [
        "https://bot.seekport.com"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "SeekportBot"
        ],
        "instances": [
          "Mozilla/5.0 (compatible; SeekportBot; +https://bot.seekport.com)"
        ]
      },
      "behavior": {
        "respects_robots_txt": true,
        "robots_token": "SeekportBot"
      },
      "verification": [
        {
          "type": "cidr_feed",
          "url": "https://bot.seekport.com/seekportbot_ips.txt",
          "format": "text_lines"
        }
      ],
      "tier": 1,
      "tier_label": "Fully verifiable",
      "first_seen": "2026-07-16T13:18:54.422Z",
      "resolved_cidrs": [
        "104.207.153.148/32",
        "104.207.154.9/32",
        "104.238.141.198/32",
        "104.248.16.236/32",
        "104.248.25.93/32",
        "135.125.245.119/32",
        "135.181.113.179/32",
        "135.181.114.163/32",
        "135.181.114.235/32",
        "135.181.119.46/32",
        "135.181.136.229/32",
        "135.181.136.240/32",
        "135.181.136.241/32",
        "135.181.137.100/32",
        "135.181.137.102/32",
        "135.181.137.110/32",
        "135.181.137.115/32",
        "135.181.137.2/32",
        "135.181.137.52/32",
        "135.181.137.58/32",
        "135.181.137.59/32",
        "135.181.137.60/32",
        "135.181.137.61/32",
        "135.181.137.62/32",
        "135.181.137.98/32",
        "135.181.137.99/32",
        "135.181.138.2/32",
        "135.181.138.3/32",
        "135.181.138.4/32",
        "135.181.138.45/32",
        "135.181.138.46/32",
        "135.181.138.47/32",
        "135.181.138.49/32",
        "135.181.138.5/32",
        "135.181.138.58/32",
        "135.181.138.6/32",
        "135.181.138.7/32",
        "135.181.138.8/32",
        "135.181.138.9/32",
        "135.181.140.112/32",
        "135.181.140.124/32",
        "135.181.140.143/32",
        "135.181.140.163/32",
        "135.181.140.173/32",
        "135.181.140.183/32",
        "135.181.140.185/32",
        "135.181.140.186/32",
        "135.181.140.19/32",
        "135.181.140.21/32",
        "135.181.140.22/32",
        "135.181.140.251/32",
        "135.181.140.252/32",
        "135.181.140.34/32",
        "135.181.140.35/32",
        "135.181.140.36/32",
        "135.181.140.38/32",
        "135.181.177.220/32",
        "135.181.3.203/32",
        "135.181.4.50/32",
        "135.181.4.54/32",
        "135.181.4.69/32",
        "135.181.61.178/32",
        "135.181.62.173/32",
        "135.181.72.179/32",
        "135.181.73.252/32",
        "135.181.74.194/32",
        "135.181.74.195/32",
        "135.181.74.196/32",
        "135.181.76.42/32",
        "135.181.78.178/32",
        "135.181.78.179/32",
        "135.181.78.180/32",
        "135.181.78.181/32",
        "135.181.78.182/32",
        "135.181.78.184/32",
        "135.181.79.114/32",
        "135.181.79.61/32",
        "135.181.79.99/32",
        "135.181.82.112/32",
        "135.181.85.126/32",
        "135.181.85.180/32",
        "135.181.85.192/32",
        "135.181.85.193/32",
        "135.181.87.126/32",
        "138.197.181.193/32",
        "141.94.104.105/32",
        "141.94.104.115/32",
        "141.94.104.138/32",
        "141.94.104.30/32",
        "141.94.105.141/32",
        "141.94.105.186/32",
        "141.94.105.203/32",
        "141.94.105.77/32",
        "141.94.105.81/32",
        "141.94.106.127/32",
        "141.94.106.21/32",
        "141.94.106.23/32",
        "141.94.106.55/32",
        "141.94.106.71/32",
        "141.94.107.146/32",
        "141.94.107.230/32",
        "141.94.107.54/32",
        "141.94.107.83/32",
        "141.94.169.120/32",
        "142.93.105.87/32",
        "142.93.168.165/32",
        "142.93.168.199/32",
        "144.202.114.74/32",
        "144.202.117.163/32",
        "144.202.119.127/32",
        "144.202.121.92/32",
        "144.202.123.60/32",
        "149.248.13.145/32",
        "149.248.13.240/32",
        "149.248.17.135/32",
        "149.248.17.24/32",
        "149.248.17.243/32",
        "149.248.21.146/32",
        "149.248.4.32/32",
        "149.28.68.230/32",
        "149.28.83.109/32",
        "149.28.85.67/32",
        "149.28.89.240/32",
        "159.223.29.210/32",
        "159.223.30.219/32",
        "163.172.129.78/32",
        "164.92.252.156/32",
        "164.92.252.7/32",
        "164.92.254.30/32",
        "165.22.23.21/32",
        "167.172.100.234/32",
        "167.172.100.236/32",
        "167.172.100.9/32",
        "167.172.102.155/32",
        "167.172.102.172/32",
        "167.172.102.189/32",
        "167.172.104.126/32",
        "167.172.104.227/32",
        "167.172.104.243/32",
        "167.172.104.245/32",
        "167.172.104.5/32",
        "167.172.104.75/32",
        "167.172.105.77/32",
        "167.172.106.119/32",
        "167.172.106.24/32",
        "167.172.106.28/32",
        "167.172.106.99/32",
        "167.172.108.165/32",
        "167.172.108.192/32",
        "167.172.108.98/32",
        "167.172.110.123/32",
        "167.172.110.160/32",
        "167.172.110.37/32",
        "167.172.177.184/32",
        "167.172.177.208/32",
        "167.172.179.142/32",
        "167.172.179.178/32",
        "167.172.179.51/32",
        "167.172.182.251/32",
        "167.172.182.89/32",
        "167.172.183.41/32",
        "167.172.183.73/32",
        "167.172.187.15/32",
        "167.172.187.173/32",
        "167.172.187.250/32",
        "167.172.190.125/32",
        "167.172.190.139/32",
        "167.172.190.4/32",
        "167.172.190.41/32",
        "167.172.190.44/32",
        "167.172.96.105/32",
        "167.172.96.192/32",
        "167.172.96.205/32",
        "167.172.96.99/32",
        "167.172.97.136/32",
        "167.172.97.4/32",
        "167.172.97.46/32",
        "167.172.98.25/32",
        "167.172.98.50/32",
        "167.71.43.145/32",
        "178.128.206.198/32",
        "198.244.143.44/32",
        "206.81.18.244/32",
        "206.81.26.33/32",
        "206.81.29.118/32",
        "207.246.100.87/32",
        "207.246.102.173/32",
        "207.246.103.10/32",
        "207.246.106.44/32",
        "207.246.109.253/32",
        "207.246.96.241/32",
        "212.47.237.184/32",
        "212.47.240.195/32",
        "212.47.242.4/32",
        "212.47.244.167/32",
        "212.47.245.230/32",
        "212.47.249.219/32",
        "212.47.251.165/32",
        "45.32.64.84/32",
        "45.32.71.141/32",
        "45.32.76.221/32",
        "45.32.79.194/32",
        "45.32.79.200/32",
        "45.32.80.202/32",
        "45.32.86.191/32",
        "45.32.86.192/32",
        "45.32.90.31/32",
        "45.32.94.222/32",
        "45.63.49.97/32",
        "45.63.53.35/32",
        "45.63.53.92/32",
        "45.63.54.35/32",
        "45.76.173.185/32",
        "45.76.173.93/32",
        "45.76.67.107/32",
        "45.76.75.127/32",
        "45.76.76.169/32",
        "45.77.122.130/32",
        "45.77.85.70/32",
        "46.101.178.140/32",
        "46.101.217.136/32",
        "51.15.128.42/32",
        "51.15.196.148/32",
        "51.15.206.253/32",
        "51.15.217.228/32",
        "51.15.218.154/32",
        "51.15.227.225/32",
        "51.15.232.133/32",
        "51.15.234.161/32",
        "51.158.107.242/32",
        "51.158.120.235/32",
        "51.158.123.235/32",
        "51.158.79.36/32",
        "51.161.80.5/32",
        "51.161.82.53/32",
        "51.161.83.127/32",
        "51.161.83.248/32",
        "51.161.83.91/32",
        "51.68.164.132/32",
        "51.68.164.224/32",
        "51.68.164.64/32",
        "51.68.164.98/32",
        "51.68.165.11/32",
        "51.68.165.184/32",
        "51.68.165.201/32",
        "51.68.165.223/32",
        "51.68.165.38/32",
        "51.68.166.168/32",
        "51.68.166.195/32",
        "51.68.166.201/32",
        "51.68.166.37/32",
        "51.68.167.111/32",
        "51.68.167.15/32",
        "51.68.167.245/32",
        "51.68.167.77/32",
        "51.68.89.88/32",
        "51.75.168.198/32",
        "51.75.62.184/32",
        "51.75.62.3/32",
        "51.77.108.244/32",
        "51.79.100.85/32",
        "51.79.103.17/32",
        "51.79.103.248/32",
        "51.79.25.241/32",
        "51.79.29.202/32",
        "51.79.88.205/32",
        "51.79.91.170/32",
        "51.79.91.233/32",
        "51.79.91.239/32",
        "51.79.91.41/32",
        "51.79.91.43/32",
        "51.79.92.212/32",
        "51.79.92.45/32",
        "51.79.93.156/32",
        "51.83.224.16/32",
        "51.83.224.179/32",
        "51.83.224.227/32",
        "51.83.224.23/32",
        "51.83.224.236/32",
        "51.83.225.218/32",
        "51.83.225.251/32",
        "51.83.225.252/32",
        "51.83.225.82/32",
        "51.83.226.105/32",
        "51.83.226.111/32",
        "51.83.226.204/32",
        "51.83.226.53/32",
        "51.83.226.9/32",
        "51.83.227.148/32",
        "51.83.227.226/32",
        "51.83.227.59/32",
        "51.83.34.104/32",
        "51.83.34.114/32",
        "51.83.34.183/32",
        "51.83.34.193/32",
        "51.83.34.224/32",
        "51.83.34.35/32",
        "51.83.34.51/32",
        "51.83.34.90/32",
        "51.83.35.127/32",
        "51.83.35.160/32",
        "51.83.35.163/32",
        "51.83.35.173/32",
        "51.83.35.207/32",
        "51.83.35.39/32",
        "51.83.35.86/32",
        "51.83.35.9/32",
        "51.83.35.92/32",
        "51.83.35.94/32",
        "51.89.224.158/32",
        "54.37.0.174/32",
        "54.37.0.218/32",
        "54.37.0.32/32",
        "54.37.0.77/32",
        "54.37.1.207/32",
        "54.37.1.234/32",
        "54.37.2.145/32",
        "54.37.2.4/32",
        "54.37.4.36/32",
        "54.37.5.1/32",
        "54.37.5.234/32",
        "54.37.6.165/32",
        "54.37.7.201/32",
        "54.37.7.218/32",
        "54.37.7.64/32",
        "54.37.77.164/32",
        "65.108.103.100/32",
        "65.108.103.101/32",
        "65.108.103.102/32",
        "65.108.103.103/32",
        "65.108.103.104/32",
        "65.108.103.105/32",
        "65.108.103.106/32",
        "65.108.103.107/32",
        "65.108.103.108/32",
        "65.108.103.109/32",
        "65.108.103.110/32",
        "65.108.103.111/32",
        "65.108.103.112/32",
        "65.108.103.113/32",
        "65.108.103.114/32",
        "65.108.103.115/32",
        "65.108.103.116/32",
        "65.108.103.50/32",
        "65.108.103.51/32",
        "65.108.103.52/32",
        "65.108.103.96/32",
        "65.108.103.97/32",
        "65.108.103.98/32",
        "65.108.103.99/32",
        "65.108.11.66/32",
        "65.108.13.103/32",
        "65.108.13.104/32",
        "65.108.13.42/32",
        "65.108.2.73/32",
        "65.108.41.10/32",
        "65.108.41.9/32",
        "65.108.44.74/32",
        "65.108.45.180/32",
        "65.108.6.231/32",
        "65.108.6.5/32",
        "65.108.6.6/32",
        "65.108.66.11/32",
        "65.108.67.8/32",
        "65.108.71.126/32",
        "65.108.72.9/32",
        "65.108.74.119/32",
        "65.108.74.120/32",
        "65.108.74.44/32",
        "65.108.75.167/32",
        "65.108.77.15/32",
        "65.108.77.16/32",
        "65.108.79.171/32",
        "65.108.8.132/32",
        "65.108.8.9/32",
        "65.108.99.107/32",
        "65.108.99.108/32",
        "65.108.99.109/32",
        "65.108.99.110/32",
        "65.108.99.111/32",
        "65.108.99.112/32",
        "65.108.99.113/32",
        "65.108.99.114/32",
        "65.108.99.115/32",
        "65.108.99.116/32",
        "65.108.99.117/32",
        "65.108.99.118/32",
        "65.108.99.119/32",
        "65.108.99.120/32",
        "65.108.99.121/32",
        "65.108.99.122/32",
        "65.108.99.123/32",
        "65.108.99.124/32",
        "65.108.99.125/32",
        "65.108.99.126/32",
        "65.108.99.172/32",
        "65.108.99.173/32",
        "65.108.99.174/32",
        "65.108.99.175/32",
        "65.108.99.176/32",
        "65.108.99.177/32",
        "65.108.99.178/32",
        "65.108.99.179/32",
        "65.108.99.180/32",
        "65.108.99.181/32",
        "65.108.99.182/32",
        "65.108.99.183/32",
        "65.108.99.184/32",
        "65.108.99.185/32",
        "65.108.99.186/32",
        "65.108.99.187/32",
        "65.108.99.188/32",
        "65.108.99.189/32",
        "65.108.99.190/32",
        "65.108.99.40/32",
        "65.108.99.41/32",
        "65.108.99.42/32",
        "65.108.99.43/32",
        "65.108.99.44/32",
        "65.108.99.45/32",
        "65.108.99.46/32",
        "65.108.99.47/32",
        "65.108.99.48/32",
        "65.108.99.49/32",
        "65.108.99.50/32",
        "65.108.99.51/32",
        "65.108.99.52/32",
        "65.108.99.53/32",
        "65.108.99.54/32",
        "65.108.99.55/32",
        "65.108.99.56/32",
        "65.108.99.57/32",
        "65.108.99.58/32",
        "65.21.136.197/32",
        "65.21.177.80/32",
        "65.21.179.1/32",
        "65.21.180.65/32",
        "65.21.197.13/32",
        "65.21.199.133/32",
        "65.21.202.117/32",
        "65.21.205.62/32",
        "65.21.206.24/32",
        "65.21.207.198/32",
        "65.21.231.77/32",
        "65.21.237.134/32",
        "65.21.237.6/32",
        "65.21.35.179/32",
        "65.21.35.180/32",
        "65.21.35.181/32",
        "65.21.35.182/32",
        "65.21.35.183/32",
        "65.21.35.184/32",
        "65.21.35.185/32",
        "65.21.35.186/32",
        "65.21.35.187/32",
        "65.21.35.188/32",
        "65.21.35.189/32",
        "65.21.35.190/32",
        "65.21.35.194/32",
        "65.21.35.195/32",
        "65.21.35.196/32",
        "65.21.35.197/32",
        "65.21.35.198/32",
        "65.21.35.199/32",
        "65.21.35.200/32",
        "65.21.35.201/32",
        "65.21.35.202/32",
        "65.21.35.203/32",
        "65.21.35.204/32",
        "65.21.35.205/32",
        "65.21.35.223/32",
        "65.21.35.224/32",
        "65.21.35.225/32",
        "65.21.35.226/32",
        "65.21.35.227/32",
        "65.21.35.228/32",
        "65.21.35.229/32",
        "65.21.35.230/32",
        "65.21.35.231/32",
        "65.21.35.232/32",
        "65.21.35.233/32",
        "65.21.35.234/32",
        "65.21.35.235/32",
        "65.21.35.236/32",
        "65.21.35.237/32",
        "65.21.35.238/32",
        "65.21.35.239/32",
        "65.21.35.240/32",
        "65.21.35.241/32",
        "65.21.35.242/32",
        "65.21.35.243/32",
        "65.21.35.244/32",
        "65.21.35.245/32",
        "65.21.35.246/32",
        "65.21.35.247/32",
        "65.21.35.248/32",
        "65.21.35.249/32",
        "65.21.35.250/32",
        "65.21.35.251/32",
        "65.21.35.252/32",
        "65.21.35.253/32",
        "65.21.35.254/32",
        "65.21.74.6/32",
        "65.21.91.185/32",
        "66.42.110.246/32",
        "66.42.97.195/32",
        "68.183.65.188/32",
        "8.6.8.194/32",
        "95.217.199.97/32",
        "95.217.203.110/32",
        "95.217.72.254/32"
      ]
    },
    {
      "id": "semanticscholar-crawler",
      "name": "SemanticScholarBot",
      "operator": {
        "name": "Allen Institute for AI (Ai2)",
        "url": "https://allenai.org"
      },
      "description": "SemanticScholarBot is the crawler for Semantic Scholar, an academic search engine built by the Allen Institute for AI. It crawls the web to find scholarly PDFs and metadata for indexing.",
      "docs": [
        "https://www.semanticscholar.org/crawler"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "SemanticScholarBot"
        ],
        "instances": [
          "SemanticScholarBot/1.0 (+http://s2.allenai.org/bot.html)",
          "Mozilla/5.0 (compatible) SemanticScholarBot (+https://www.semanticscholar.org/crawler)"
        ]
      },
      "behavior": {
        "respects_robots_txt": false
      },
      "verification": [],
      "tier": 3,
      "tier_label": "Listed only",
      "first_seen": "2026-07-16T00:59:46.504Z",
      "resolved_cidrs": []
    },
    {
      "id": "seznambot",
      "name": "SeznamBot",
      "operator": {
        "name": "Seznam.cz",
        "url": "https://www.seznam.cz"
      },
      "description": "Seznam's web crawler, fetching pages for the Seznam.cz search engine used primarily in the Czech Republic.",
      "docs": [
        "https://o-seznam.cz/napoveda/vyhledavani/en/seznambot-crawler/"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "SeznamBot/\\d"
        ],
        "instances": [
          "Mozilla/5.0 (compatible; SeznamBot/4.0; +https://o-seznam.cz/napoveda/vyhledavani/en/seznambot-crawler/)",
          "Mozilla/5.0 (compatible; SeznamBot/3.2; +http://napoveda.seznam.cz/en/seznambot-intro/)"
        ]
      },
      "behavior": {
        "respects_robots_txt": true,
        "robots_token": "SeznamBot"
      },
      "verification": [
        {
          "type": "rdns",
          "masks": [
            "*.seznam.cz"
          ]
        }
      ],
      "tier": 2,
      "tier_label": "Verifiable",
      "first_seen": "2026-07-08T19:13:49.571Z",
      "resolved_cidrs": []
    },
    {
      "id": "sogou-crawler",
      "name": "Sogou Spider",
      "operator": {
        "name": "Sogou",
        "url": "https://www.sogou.com/"
      },
      "description": "Sogou Spider is the web crawler for Sogou, a major Chinese search engine (owned by Tencent). It crawls and indexes web pages, news, and images to build Sogou's search index.",
      "docs": [
        "https://www.sogou.com/docs/help/webmasters.htm"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "Sogou"
        ],
        "instances": [
          "Sogou News Spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)",
          "Sogou Pic Spider/3.0(+http://www.sogou.com/docs/help/webmasters.htm#07)",
          "Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)"
        ]
      },
      "behavior": {
        "respects_robots_txt": true
      },
      "verification": [],
      "tier": 3,
      "tier_label": "Listed only",
      "first_seen": "2026-07-16T00:59:46.504Z",
      "resolved_cidrs": []
    },
    {
      "id": "stepstone-crawler",
      "name": "stepstoneCrawlBot",
      "operator": {
        "name": "The Stepstone Group",
        "url": "https://www.thestepstonegroup.com"
      },
      "description": "The Stepstone Group's job-listing crawler. Its crawler page states that it processes only publicly available information, observes robots.txt directives and keeps intervals between requests to avoid loading servers, and publishes the user agent it sends for transparency.",
      "docs": [
        "https://www.thestepstonegroup.com/english/crawler/"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "stepstoneCrawlBot"
        ],
        "instances": [
          "Mozilla/5.0 (compatible; stepstoneCrawlBot; +https://www.thestepstonegroup.com/english/crawler/)"
        ]
      },
      "behavior": {
        "respects_robots_txt": true
      },
      "verification": [],
      "tier": 3,
      "tier_label": "Listed only",
      "first_seen": "2026-08-23T13:20:46.873Z",
      "resolved_cidrs": []
    },
    {
      "id": "stractbot",
      "name": "StractBot",
      "operator": {
        "name": "Stract",
        "url": "https://stract.com"
      },
      "description": "The crawler for Stract, an open-source search engine. Stract documents the crawler's user agent and robots.txt behavior but publishes no IP list or reverse-DNS verification method.",
      "docs": [
        "https://stract.com/bot"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "StractBot/\\d"
        ],
        "instances": [
          "Mozilla/5.0 (compatible; StractBot/0.2; open source search engine; +https://stract.com/webmasters)"
        ]
      },
      "behavior": {
        "respects_robots_txt": true,
        "robots_token": "StractBot"
      },
      "verification": [],
      "tier": 3,
      "tier_label": "Listed only",
      "first_seen": "2026-07-08T19:13:49.571Z",
      "resolved_cidrs": []
    },
    {
      "id": "tineye-crawler",
      "name": "TinEye",
      "operator": {
        "name": "TinEye",
        "url": "https://tineye.com"
      },
      "description": "TinEye-bot is the web crawler operated by TinEye, the reverse image search engine built by Idee Inc. It crawls the web to discover and index images for TinEye's image-matching search service.",
      "docs": [
        "https://www.tineye.com/crawler.html"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "TinEye"
        ],
        "instances": [
          "Mozilla/5.0 (compatible; TinEye-bot/1.31; +http://www.tineye.com/crawler.html)",
          "TinEye/1.1 (http://tineye.com/crawler.html)"
        ]
      },
      "behavior": {
        "respects_robots_txt": false
      },
      "verification": [],
      "tier": 3,
      "tier_label": "Listed only",
      "first_seen": "2026-07-16T01:14:20.999Z",
      "resolved_cidrs": []
    },
    {
      "id": "trovitbot",
      "name": "TrovitBot",
      "operator": {
        "name": "Trovit",
        "url": "https://www.trovit.com"
      },
      "description": "Trovit's web crawler, which discovers new and updated pages to add to the Trovit classifieds search index. Trovit documents that it does not fetch most sites more than once per second and that it can be blocked with the trovitBot robots.txt token.",
      "docs": [
        "https://www.trovit.com/bot.html"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "trovitBot"
        ],
        "instances": [
          "Mozilla/5.0 (compatible; trovitBot 1.0; +http://www.trovit.com/bot.html)"
        ]
      },
      "behavior": {
        "respects_robots_txt": true,
        "robots_token": "trovitBot"
      },
      "verification": [],
      "tier": 3,
      "tier_label": "Listed only",
      "first_seen": "2026-07-28T19:51:24.263Z",
      "resolved_cidrs": []
    },
    {
      "id": "webzio",
      "name": "Webzio",
      "operator": {
        "name": "Webz.io Ltd.",
        "url": "https://webz.io"
      },
      "description": "Crawler for Webz.io (formerly Webhose.io), a web-data provider that collects publicly available content to build structured data feeds. Webz.io publishes no IP ranges, so requests cannot be verified.",
      "docs": [
        "https://webz.io/bot.html"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "webzio\\s"
        ],
        "instances": [
          "webzio (+https://webz.io/bot.html)"
        ]
      },
      "behavior": {
        "respects_robots_txt": true
      },
      "verification": [],
      "tier": 3,
      "tier_label": "Listed only",
      "first_seen": "2026-07-20T20:56:47.446Z",
      "resolved_cidrs": []
    },
    {
      "id": "yahoo-crawler",
      "name": "Yahoo! Slurp",
      "operator": {
        "name": "Yahoo",
        "url": "https://www.yahoo.com/"
      },
      "description": "Slurp is Yahoo Search's web crawler. It crawls and indexes pages for Yahoo Search results and also gathers content for Yahoo News, Finance, and Sports.",
      "docs": [
        "https://help.yahoo.com/kb/SLN22600.html"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "Slurp"
        ],
        "instances": [
          "Mozilla/5.0 (compatible; Yahoo! Slurp/3.0; http://help.yahoo.com/help/us/ysearch/slurp)",
          "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
        ]
      },
      "behavior": {
        "respects_robots_txt": true,
        "robots_token": "Slurp"
      },
      "verification": [],
      "tier": 3,
      "tier_label": "Listed only",
      "first_seen": "2026-07-16T00:59:46.504Z",
      "resolved_cidrs": []
    },
    {
      "id": "yahoo-crawler-japan",
      "name": "Yahoo! JAPAN Crawler",
      "operator": {
        "name": "LY Corporation (Yahoo! JAPAN)",
        "url": "https://www.yahoo.co.jp/"
      },
      "description": "Web crawler operated by Yahoo! JAPAN (LY Corporation) to collect pages for its Japanese search index and related services. It identifies itself with Y!J-prefixed user-agent tokens and follows the Robots Exclusion Protocol.",
      "docs": [
        "https://info-search.yahoo.co.jp/crawl/"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "Y!J-"
        ],
        "instances": [
          "Y!J-ASR/0.1 crawler (http://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/)",
          "Y!J-BRJ/YATS crawler (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)"
        ]
      },
      "behavior": {
        "respects_robots_txt": true
      },
      "verification": [],
      "tier": 3,
      "tier_label": "Listed only",
      "first_seen": "2026-07-16T01:14:20.999Z",
      "resolved_cidrs": []
    },
    {
      "id": "yandex-blogs",
      "name": "YandexBlogs",
      "operator": {
        "name": "Yandex",
        "url": "https://yandex.com"
      },
      "description": "Yandex's blog-search robot. Yandex's robot table documents it as the blog search robot that indexes post comments, and lists it as following robots.txt directives.",
      "docs": [
        "https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.html"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "YandexBlogs/[\\d.]+"
        ],
        "instances": [
          "Mozilla/5.0 (compatible; YandexBlogs/0.99; robot; +http://yandex.com/bots)"
        ]
      },
      "behavior": {
        "respects_robots_txt": true,
        "robots_token": "YandexBlogs"
      },
      "verification": [
        {
          "type": "rdns",
          "masks": [
            "*.yandex.ru",
            "*.yandex.net",
            "*.yandex.com"
          ]
        }
      ],
      "tier": 2,
      "tier_label": "Verifiable",
      "first_seen": "2026-08-06T15:22:58.093Z",
      "resolved_cidrs": []
    },
    {
      "id": "yandex-crawler-javascript",
      "name": "YandexRenderResourcesBot",
      "operator": {
        "name": "Yandex",
        "url": "https://yandex.com"
      },
      "description": "Yandex robot that loads resources such as JavaScript and CSS needed to render pages during crawling for Yandex Search.",
      "docs": [
        "https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.html"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "YandexRenderResourcesBot/\\d"
        ],
        "instances": [
          "Mozilla/5.0 (compatible; YandexRenderResourcesBot/1.0; +http://yandex.com/bots) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0"
        ]
      },
      "behavior": {
        "respects_robots_txt": false
      },
      "verification": [
        {
          "type": "rdns",
          "masks": [
            "*.yandex.ru",
            "*.yandex.net",
            "*.yandex.com"
          ]
        }
      ],
      "tier": 2,
      "tier_label": "Verifiable",
      "first_seen": "2026-07-16T00:59:46.504Z",
      "resolved_cidrs": []
    },
    {
      "id": "yandex-favicons",
      "name": "YandexFavicons",
      "operator": {
        "name": "Yandex",
        "url": "https://yandex.com"
      },
      "description": "Yandex's favicon fetcher. Yandex's robot table documents it as downloading a site's favicon file for display in search results, and lists it as one of the Yandex robots that do not follow robots.txt directives.",
      "docs": [
        "https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.html"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "YandexFavicons/\\d"
        ],
        "instances": [
          "Mozilla/5.0 (compatible; YandexFavicons/1.0; +http://yandex.com/bots)"
        ]
      },
      "behavior": {
        "respects_robots_txt": false
      },
      "verification": [
        {
          "type": "rdns",
          "masks": [
            "*.yandex.ru",
            "*.yandex.net",
            "*.yandex.com"
          ]
        }
      ],
      "tier": 2,
      "tier_label": "Verifiable",
      "first_seen": "2026-08-06T15:22:58.093Z",
      "resolved_cidrs": []
    },
    {
      "id": "yandex-images",
      "name": "YandexImages",
      "operator": {
        "name": "Yandex",
        "url": "https://yandex.com"
      },
      "description": "Yandex's image-indexing robot, which fetches images so they can be displayed in Yandex Images. Yandex documents it as a separate robot from YandexBot and lists it as following robots.txt directives.",
      "docs": [
        "https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.html"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "YandexImages/\\d"
        ],
        "instances": [
          "Mozilla/5.0 (compatible; YandexImages/3.0; +http://yandex.com/bots)"
        ]
      },
      "behavior": {
        "respects_robots_txt": true,
        "robots_token": "YandexImages"
      },
      "verification": [
        {
          "type": "rdns",
          "masks": [
            "*.yandex.ru",
            "*.yandex.net",
            "*.yandex.com"
          ]
        }
      ],
      "tier": 2,
      "tier_label": "Verifiable",
      "first_seen": "2026-07-28T19:51:24.263Z",
      "resolved_cidrs": []
    },
    {
      "id": "yandex-media",
      "name": "YandexMedia",
      "operator": {
        "name": "Yandex",
        "url": "https://yandex.com"
      },
      "description": "Yandex's multimedia-indexing robot. Yandex's robot table documents it as a separate robot from YandexBot, describes it as indexing multimedia data, and lists it as following robots.txt directives.",
      "docs": [
        "https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.html"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "YandexMedia/\\d"
        ],
        "instances": [
          "Mozilla/5.0 (compatible; YandexMedia/3.0; +http://yandex.com/bots)"
        ]
      },
      "behavior": {
        "respects_robots_txt": true,
        "robots_token": "YandexMedia"
      },
      "verification": [
        {
          "type": "rdns",
          "masks": [
            "*.yandex.ru",
            "*.yandex.net",
            "*.yandex.com"
          ]
        }
      ],
      "tier": 2,
      "tier_label": "Verifiable",
      "first_seen": "2026-08-06T15:22:58.093Z",
      "resolved_cidrs": []
    },
    {
      "id": "yandex-video",
      "name": "YandexVideo",
      "operator": {
        "name": "Yandex",
        "url": "https://yandex.com"
      },
      "description": "Yandex's video-indexing robot, which fetches pages and video content for display in Yandex video search. Yandex documents it as a separate robot from YandexBot and lists it as following robots.txt directives.",
      "docs": [
        "https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.html"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "YandexVideo/\\d"
        ],
        "instances": [
          "Mozilla/5.0 (compatible; YandexVideo/3.0; +http://yandex.com/bots)"
        ]
      },
      "behavior": {
        "respects_robots_txt": true,
        "robots_token": "YandexVideo"
      },
      "verification": [
        {
          "type": "rdns",
          "masks": [
            "*.yandex.ru",
            "*.yandex.net",
            "*.yandex.com"
          ]
        }
      ],
      "tier": 2,
      "tier_label": "Verifiable",
      "first_seen": "2026-07-28T19:51:24.263Z",
      "resolved_cidrs": []
    },
    {
      "id": "yandexbot",
      "name": "YandexBot",
      "operator": {
        "name": "Yandex",
        "url": "https://yandex.com"
      },
      "description": "Yandex's primary web crawler, fetching pages for Yandex Search indexing.",
      "docs": [
        "https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.html"
      ],
      "category": "search-engine",
      "user_agents": {
        "patterns": [
          "YandexBot/\\d"
        ],
        "instances": [
          "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)",
          "Mozilla/5.0 (compatible; YandexBot/3.0; MirrorDetector; +http://yandex.com/bots)"
        ]
      },
      "behavior": {
        "respects_robots_txt": true,
        "robots_token": "YandexBot"
      },
      "verification": [
        {
          "type": "rdns",
          "masks": [
            "*.yandex.ru",
            "*.yandex.net",
            "*.yandex.com"
          ]
        }
      ],
      "tier": 2,
      "tier_label": "Verifiable",
      "first_seen": "2026-07-08T19:13:49.571Z",
      "resolved_cidrs": []
    }
  ]
}