{ "project": "Verified Bots Directory", "generated_at": "2026-08-23T13:31:38.100Z", "count": 11, "bots": [ { "id": "amzn-user", "name": "Amzn-User", "operator": { "name": "Amazon", "url": "https://www.amazon.com" }, "description": "Amazon's on-demand fetcher supporting user actions, such as responding to Alexa queries that need up-to-date information; Amazon states it does not crawl content for generative AI model training. Amazon publishes a human-readable live-crawl IP list but no machine-parseable feed.", "docs": [ "https://developer.amazon.com/amazonbot" ], "category": "ai-assistant", "user_agents": { "patterns": [ "Amzn-User/\\d" ], "instances": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amzn-User/0.1) Chrome/W.X.Y.Z Safari/537.36" ] }, "behavior": { "respects_robots_txt": true, "robots_token": "Amzn-User" }, "verification": [], "tier": 3, "tier_label": "Listed only", "first_seen": "2026-07-26T13:26:40.725Z", "resolved_cidrs": [] }, { "id": "chatgpt-user", "name": "ChatGPT-User", "operator": { "name": "OpenAI", "url": "https://openai.com" }, "description": "OpenAI's user-triggered fetcher, used when a ChatGPT user or a GPT Action asks the assistant to visit a specific page. It does not crawl autonomously.", "docs": [ "https://developers.openai.com/api/docs/bots" ], "category": "ai-assistant", "user_agents": { "patterns": [ "ChatGPT-User/\\d" ], "instances": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot" ] }, "behavior": { "respects_robots_txt": false, "robots_token": "ChatGPT-User" }, "verification": [ { "type": "cidr_feed", "url": "https://openai.com/chatgpt-user.json", "format": "prefixes" } ], "tier": 1, "tier_label": "Fully verifiable", "first_seen": "2026-07-08T19:13:49.571Z", "resolved_cidrs": [ "104.208.184.192/28", "104.210.139.192/28", "104.210.139.224/28", "128.85.198.32/28", "13.65.138.112/28", "13.67.72.16/28", "13.71.2.208/28", "13.76.116.80/28", "13.76.32.208/28", "13.83.167.128/28", "13.83.237.176/28", "132.196.82.48/28", "134.149.233.80/28", "135.116.136.160/28", "135.13.64.240/28", "135.220.73.208/28", "135.220.73.240/28", "135.237.131.208/28", "135.237.133.48/28", "137.135.191.176/28", "138.91.46.96/28", "145.132.136.96/28", "158.158.5.32/28", "168.63.252.240/28", "172.162.248.64/28", "172.170.1.80/28", "172.170.225.0/28", "172.170.241.80/28", "172.175.152.224/28", "172.178.140.144/28", "172.178.141.112/28", "172.178.141.128/28", "172.183.143.224/28", "172.183.222.128/28", "172.192.112.208/28", "172.197.160.192/28", "172.197.203.16/28", "172.199.137.80/28", "172.204.28.224/28", "172.204.96.80/28", "172.205.189.192/28", "172.207.1.32/28", "172.212.172.160/28", "172.215.215.32/28", "191.233.199.160/28", "191.237.249.64/28", "20.102.212.144/28", "20.113.211.112/28", "20.113.225.112/28", "20.125.112.224/28", "20.125.144.144/28", "20.161.75.208/28", "20.168.7.192/28", "20.169.73.32/28", "20.169.73.64/28", "20.169.78.128/28", "20.169.78.160/28", "20.169.78.192/28", "20.169.78.48/28", "20.169.78.64/28", "20.169.86.224/28", "20.169.86.240/28", "20.170.184.16/28", "20.170.184.32/28", "20.170.184.48/28", "20.170.184.64/28", "20.170.184.80/28", "20.172.29.32/28", "20.199.211.160/28", "20.199.242.0/28", "20.200.212.240/28", "20.210.211.192/28", "20.215.187.208/28", "20.215.219.128/28", "20.215.219.160/28", "20.215.219.208/28", "20.218.30.240/28", "20.219.71.192/28", "20.222.36.192/28", "20.227.140.32/28", "20.228.106.176/28", "20.235.87.224/28", "20.249.63.208/28", "20.250.136.64/28", "20.250.6.128/28", "20.45.178.144/28", "20.55.229.144/28", "20.57.199.192/28", "20.63.180.96/28", "20.63.221.64/28", "20.83.243.176/28", "23.102.140.144/28", "23.102.141.32/28", "23.98.142.176/28", "23.98.186.176/28", "23.98.186.192/28", "23.98.186.64/28", "23.98.186.96/28", "4.151.119.48/28", "4.151.71.176/28", "4.189.118.208/28", "4.189.119.48/28", "4.197.115.112/28", "4.197.22.112/28", "4.197.64.0/28", "4.197.64.16/28", "4.197.64.48/28", "4.197.64.64/28", "4.201.232.64/28", "4.205.128.176/28", "4.218.24.64/28", "4.226.200.16/28", "4.226.226.32/28", "40.116.73.208/28", "40.74.200.208/28", "40.81.234.144/28", "40.81.67.96/28", "40.84.221.208/28", "40.84.221.224/28", "48.221.184.80/28", "48.221.184.96/28", "48.221.40.176/28", "51.116.2.80/28", "51.116.221.96/28", "51.57.0.96/28", "51.59.24.64/28", "51.59.24.80/28", "51.59.48.80/28", "51.8.155.112/28", "51.8.155.48/28", "52.148.129.32/28", "52.156.77.144/28", "52.159.227.32/28", "52.159.249.96/28", "52.161.49.96/28", "52.165.212.48/28", "52.172.129.160/28", "52.173.219.112/28", "52.173.219.96/28", "52.173.221.16/28", "52.173.221.176/28", "52.173.221.208/28", "52.173.234.16/28", "52.173.234.80/28", "52.183.217.240/28", "52.190.137.144/28", "52.190.137.16/28", "52.190.139.48/28", "52.190.142.64/28", "52.190.190.16/28", "52.225.75.208/28", "52.231.30.48/28", "52.231.34.176/28", "52.231.39.144/28", "52.231.50.64/28", "52.236.94.144/28", "52.241.146.208/28", "52.242.132.224/28", "52.242.132.240/28", "52.255.109.112/28", "52.255.109.144/28", "52.255.109.80/28", "52.255.109.96/28", "52.255.111.0/28", "52.255.111.80/28", "57.154.174.112/28", "57.154.175.0/28", "57.154.187.32/28", "68.154.28.96/28", "68.220.57.64/28", "70.153.189.192/28", "70.153.32.16/28", "70.153.32.32/28", "70.153.87.224/28", "70.156.152.96/28", "74.161.200.96/28", "74.224.217.64/28", "74.226.253.160/28", "74.7.35.112/28", "74.7.35.48/28", "74.7.36.64/28", "74.7.36.80/28", "74.7.36.96/28", "85.211.128.16/28", "85.211.128.32/28", "9.129.0.0/17", "9.160.128.16/28", "9.160.128.64/28", "9.160.163.128/28", "9.160.34.144/28", "9.160.36.16/28", "9.160.96.16/28", "9.163.101.48/28", "9.205.128.32/28", "9.205.128.48/28", "9.205.25.128/28", "9.205.30.128/28", "9.205.30.144/28", "9.205.30.176/28", "9.205.8.48/28", "9.205.8.64/28", "9.223.181.208/28", "9.234.96.192/28", "9.234.97.96/28" ] }, { "id": "claude-user", "name": "Claude-User", "operator": { "name": "Anthropic", "url": "https://www.anthropic.com" }, "description": "Anthropic's user-triggered fetcher, used when a person asks Claude to visit a specific web page (e.g. via tool use in a conversation).", "docs": [ "https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler" ], "category": "ai-assistant", "user_agents": { "patterns": [ "Claude-User/\\d" ], "instances": [ "Mozilla/5.0 (compatible; Claude-User/1.0; +Claude-User@anthropic.com)" ] }, "behavior": { "respects_robots_txt": false, "robots_token": "Claude-User" }, "verification": [ { "type": "cidr_feed", "url": "https://claude.com/crawling/bots.json", "format": "prefixes" } ], "tier": 1, "tier_label": "Fully verifiable", "first_seen": "2026-07-08T19:13:49.571Z", "resolved_cidrs": [ "136.107.176.208/32", "16.58.26.69/32", "18.225.238.228/32", "18.227.226.255/32", "20.102.46.224/28", "20.64.57.208/28", "216.73.216.0/22", "34.11.34.31/32", "34.150.241.79/32", "34.162.191.81/32", "34.162.230.222/32", "34.162.244.71/32", "34.182.140.95/32", "34.182.161.143/32", "34.182.218.27/32", "34.182.220.85/32", "34.182.222.37/32", "34.182.225.167/32", "34.182.226.151/32", "34.182.226.221/32", "34.186.108.163/32", "34.85.172.162/32", "35.221.29.174/32", "35.245.175.129/32", "35.245.89.239/32", "40.124.101.48/28" ] }, { "id": "duckassistbot", "name": "DuckAssistBot", "operator": { "name": "DuckDuckGo", "url": "https://duckduckgo.com" }, "description": "DuckDuckGo's real-time fetcher for DuckDuckGo Search's AI-assisted answers, which cite their sources. DuckDuckGo states the data is not used to train AI models, and publishes a machine-readable list of the bot's IP addresses.", "docs": [ "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot/" ], "category": "ai-assistant", "user_agents": { "patterns": [ "DuckAssistBot/\\d" ], "instances": [ "DuckAssistBot/1.2; (+http://duckduckgo.com/duckassistbot.html)" ] }, "behavior": { "respects_robots_txt": true, "robots_token": "DuckAssistBot" }, "verification": [ { "type": "cidr_feed", "url": "https://duckduckgo.com/duckassistbot.json", "format": "prefixes" } ], "tier": 1, "tier_label": "Fully verifiable", "first_seen": "2026-07-26T13:26:40.725Z", "resolved_cidrs": [ "104.43.54.127/32", "104.43.55.116/32", "104.43.55.117/32", "104.43.55.166/32", "104.43.55.167/32", "108.141.83.74/32", "128.203.132.152/32", "128.203.181.79/32", "128.203.214.241/32", "128.203.236.110/32", "128.203.82.154/32", "13.86.35.212/32", "13.89.106.77/32", "13.90.178.139/32", "130.107.228.224/32", "131.145.39.118/32", "131.189.183.241/32", "132.164.209.198/32", "132.196.106.146/32", "132.220.1.71/32", "134.33.207.189/32", "135.233.63.237/32", "135.234.221.112/32", "135.235.183.95/32", "172.168.115.250/32", "172.168.12.93/32", "172.168.133.76/32", "172.168.137.102/32", "172.168.152.88/32", "172.168.173.230/32", "172.168.190.186/32", "172.168.227.120/32", "172.168.254.119/32", "172.168.43.43/32", "172.168.51.216/32", "172.168.53.53/32", "172.168.81.217/32", "172.168.94.160/32", "172.169.164.109/32", "172.169.17.165/32", "172.169.177.131/32", "172.169.181.237/32", "172.169.28.184/32", "172.169.36.43/32", "172.169.53.208/32", "172.169.60.134/32", "172.169.7.33/32", "172.169.93.148/32", "172.171.37.221/32", "172.179.105.87/32", "172.179.128.118/32", "172.189.40.18/32", "172.193.184.42/32", "172.193.197.146/32", "172.193.245.229/32", "172.194.141.49/32", "172.199.55.212/32", "172.210.111.68/32", "191.233.3.197/32", "191.233.3.202/32", "191.234.216.178/32", "191.234.216.4/32", "191.235.201.214/32", "191.235.202.38/32", "191.235.202.48/32", "20.100.136.36/32", "20.100.140.155/32", "20.101.17.173/32", "20.102.34.205/32", "20.109.129.162/32", "20.109.130.190/32", "20.109.241.18/32", "20.112.58.44/32", "20.113.14.159/32", "20.113.3.121/32", "20.115.181.187/32", "20.118.11.251/32", "20.119.127.146/32", "20.12.141.99/32", "20.12.249.123/32", "20.12.252.21/32", "20.121.75.32/32", "20.13.44.19/32", "20.15.142.178/32", "20.166.171.150/32", "20.170.75.54/32", "20.175.232.228/32", "20.185.79.15/32", "20.185.79.47/32", "20.190.21.172/32", "20.191.44.119/32", "20.191.44.16/32", "20.191.44.22/32", "20.191.44.234/32", "20.191.45.212/32", "20.193.12.126/32", "20.193.24.10/32", "20.193.24.251/32", "20.193.25.197/32", "20.193.27.215/32", "20.193.45.113/32", "20.195.108.47/32", "20.195.58.189/32", "20.197.209.11/32", "20.197.209.27/32", "20.198.132.126/32", "20.198.152.208/32", "20.201.15.208/32", "20.204.228.200/32", "20.204.240.172/32", "20.204.241.148/32", "20.204.242.101/32", "20.204.242.19/32", "20.204.243.55/32", "20.204.246.254/32", "20.204.246.81/32", "20.207.107.181/32", "20.207.117.66/32", "20.207.72.11/32", "20.207.72.110/32", "20.207.72.113/32", "20.207.72.21/32", "20.207.97.190/32", "20.207.99.197/32", "20.212.139.34/32", "20.212.90.107/32", "20.216.200.223/32", "20.219.43.246/32", "20.219.45.190/32", "20.219.45.67/32", "20.221.10.115/32", "20.221.84.185/32", "20.226.133.105/32", "20.23.88.231/32", "20.231.254.179/32", "20.232.249.95/32", "20.232.51.46/32", "20.236.195.196/32", "20.236.198.192/32", "20.236.227.113/32", "20.236.233.23/32", "20.236.247.180/32", "20.236.40.93/32", "20.237.1.51/32", "20.241.135.19/32", "20.241.147.151/32", "20.241.153.203/32", "20.241.164.5/32", "20.241.241.100/32", "20.242.133.53/32", "20.242.135.165/32", "20.242.211.139/32", "20.242.215.53/32", "20.242.237.240/32", "20.246.132.168/32", "20.246.225.34/32", "20.247.235.11/32", "20.250.51.113/32", "20.252.16.193/32", "20.252.41.18/32", "20.252.63.162/32", "20.252.88.10/32", "20.252.9.6/32", "20.253.114.51/32", "20.253.59.76/32", "20.253.96.199/32", "20.29.149.44/32", "20.29.154.56/32", "20.3.1.178/32", "20.3.60.212/32", "20.40.133.240/32", "20.40.147.172/32", "20.43.150.85/32", "20.43.150.93/32", "20.43.152.28/32", "20.43.172.120/32", "20.44.222.1/32", "20.49.129.236/32", "20.49.136.28/32", "20.50.168.91/32", "20.50.48.159/32", "20.50.48.192/32", "20.50.49.0/32", "20.50.49.237/32", "20.50.49.25/32", "20.50.49.40/32", "20.50.49.55/32", "20.50.50.118/32", "20.50.50.121/32", "20.50.50.123/32", "20.50.50.130/32", "20.50.50.134/32", "20.50.50.145/32", "20.50.50.146/32", "20.50.50.163/32", "20.50.50.46/32", "20.53.134.160/32", "20.53.78.106/32", "20.53.78.123/32", "20.53.78.138/32", "20.53.78.144/32", "20.53.78.236/32", "20.53.91.2/32", "20.53.92.211/32", "20.54.224.39/32", "20.56.197.58/32", "20.56.197.63/32", "20.59.0.65/32", "20.61.142.192/32", "20.61.34.40/32", "20.62.224.44/32", "20.64.185.45/32", "20.69.129.205/32", "20.69.131.45/32", "20.69.138.149/32", "20.69.138.189/32", "20.69.144.137/32", "20.69.146.171/32", "20.69.147.21/32", "20.69.171.204/32", "20.69.74.250/32", "20.71.12.143/32", "20.71.69.210/32", "20.72.197.110/32", "20.72.203.250/32", "20.72.218.29/32", "20.72.221.33/32", "20.72.221.45/32", "20.72.225.13/32", "20.72.242.209/32", "20.72.242.93/32", "20.73.132.240/32", "20.73.202.147/32", "20.75.144.152/32", "20.75.254.61/32", "20.77.146.108/32", "20.79.226.26/32", "20.79.238.198/32", "20.79.239.66/32", "20.8.252.26/32", "20.80.114.203/32", "20.80.120.41/32", "20.80.129.80/32", "20.80.135.7/32", "20.80.156.88/32", "20.81.115.13/32", "20.82.24.207/32", "20.84.138.61/32", "20.84.219.45/32", "20.84.251.223/32", "20.85.138.75/32", "20.85.167.56/32", "20.85.187.146/32", "20.88.168.65/32", "20.9.114.242/32", "20.93.28.24/32", "20.93.49.96/32", "20.99.154.113/32", "20.99.255.235/32", "4.144.182.50/32", "4.149.142.7/32", "4.149.159.102/32", "4.149.159.71/32", "4.149.216.220/32", "4.150.142.218/32", "4.154.193.167/32", "4.156.136.79/32", "4.156.154.107/32", "4.156.30.123/32", "4.157.77.76/32", "4.172.49.103/32", "4.177.16.196/32", "4.182.10.198/32", "4.182.131.108/32", "4.195.133.120/32", "4.201.125.59/32", "4.201.141.71/32", "4.201.197.203/32", "4.201.206.133/32", "4.201.220.8/32", "4.207.220.92/32", "4.209.224.56/32", "4.213.46.14/32", "4.226.40.135/32", "4.228.76.163/32", "4.237.155.54/32", "4.237.244.80/32", "4.242.119.13/32", "4.246.10.252/32", "4.248.234.26/32", "4.249.216.104/32", "4.249.223.51/32", "4.249.238.11/32", "4.254.123.56/32", "4.255.35.121/32", "4.255.39.80/32", "40.114.182.153/32", "40.114.182.172/32", "40.114.182.45/32", "40.114.183.196/32", "40.114.183.251/32", "40.114.183.88/32", "40.119.232.146/32", "40.119.232.215/32", "40.119.232.218/32", "40.119.232.251/32", "40.119.232.50/32", "40.121.121.54/32", "40.127.154.196/32", "40.64.105.247/32", "40.64.106.11/32", "40.64.106.152/32", "40.64.67.220/32", "40.64.78.45/32", "40.64.88.145/32", "40.65.78.86/32", "40.76.162.17/32", "40.76.162.191/32", "40.76.162.208/32", "40.76.162.247/32", "40.76.163.23/32", "40.76.163.7/32", "40.76.173.151/32", "40.81.250.205/32", "40.82.218.203/32", "40.88.195.62/32", "40.88.21.235/32", "40.89.243.175/32", "48.192.16.86/32", "48.192.74.178/32", "48.200.33.227/32", "48.202.155.200/32", "48.202.171.124/32", "48.204.32.87/32", "48.206.18.152/32", "48.217.129.210/32", "48.217.212.89/32", "48.217.23.236/32", "48.223.209.110/32", "51.104.144.31/32", "51.104.146.225/32", "51.104.146.235/32", "51.104.160.167/32", "51.104.160.177/32", "51.104.161.32/32", "51.104.162.149/32", "51.104.163.250/32", "51.104.164.109/32", "51.104.164.147/32", "51.104.164.189/32", "51.104.164.215/32", "51.104.166.111/32", "51.104.167.104/32", "51.104.167.110/32", "51.104.167.19/32", "51.104.167.52/32", "51.104.167.54/32", "51.104.167.61/32", "51.104.167.71/32", "51.104.167.87/32", "51.104.167.88/32", "51.104.167.95/32", "51.104.167.96/32", "51.104.180.26/32", "51.104.180.47/32", "51.104.180.53/32", "51.107.40.209/32", "51.116.131.221/32", "51.120.48.122/32", "51.138.90.161/32", "51.138.90.206/32", "51.138.90.233/32", "51.143.114.45/32", "51.145.185.16/32", "51.8.253.152/32", "51.8.31.28/32", "51.8.71.117/32", "52.142.24.149/32", "52.142.26.175/32", "52.143.241.111/32", "52.143.242.6/32", "52.143.243.117/32", "52.143.244.81/32", "52.143.247.235/32", "52.143.95.162/32", "52.143.95.204/32", "52.146.57.51/32", "52.146.58.236/32", "52.146.59.12/32", "52.146.59.154/32", "52.146.59.156/32", "52.146.63.80/32", "52.148.161.87/32", "52.148.165.38/32", "52.149.25.43/32", "52.149.28.18/32", "52.149.28.83/32", "52.149.30.45/32", "52.149.56.151/32", "52.149.58.139/32", "52.149.58.173/32", "52.149.58.27/32", "52.149.58.69/32", "52.149.60.38/32", "52.149.61.51/32", "52.154.169.200/32", "52.154.169.50/32", "52.154.170.113/32", "52.154.170.117/32", "52.154.170.122/32", "52.154.170.209/32", "52.154.170.229/32", "52.154.170.243/32", "52.154.170.26/32", "52.154.170.28/32", "52.154.170.88/32", "52.154.170.96/32", "52.154.171.0/32", "52.154.171.150/32", "52.154.171.196/32", "52.154.171.205/32", "52.154.171.235/32", "52.154.171.250/32", "52.154.171.44/32", "52.154.171.70/32", "52.154.171.87/32", "52.154.172.2/32", "52.154.204.93/32", "52.154.60.82/32", "52.185.93.208/32", "52.186.37.211/32", "52.188.89.106/32", "52.189.70.103/32", "52.190.37.160/32", "52.191.222.158/32", "52.191.82.242/32", "52.224.147.21/32", "52.224.16.221/32", "52.224.16.229/32", "52.224.19.152/32", "52.224.191.245/32", "52.224.20.174/32", "52.224.20.181/32", "52.224.20.186/32", "52.224.20.190/32", "52.224.20.193/32", "52.224.20.203/32", "52.224.20.204/32", "52.224.20.223/32", "52.224.20.227/32", "52.224.20.249/32", "52.224.21.19/32", "52.224.21.20/32", "52.224.21.23/32", "52.224.21.27/32", "52.224.21.4/32", "52.224.21.49/32", "52.224.21.51/32", "52.224.21.53/32", "52.224.21.55/32", "52.224.21.61/32", "52.242.224.168/32", "52.250.46.221/32", "57.151.44.140/32", "57.152.17.237/32", "57.152.62.107/32", "57.152.72.128/32", "57.155.18.44/32", "57.165.205.172/32", "57.167.88.38/32", "64.236.118.43/32", "64.236.15.33/32", "64.236.93.100/32", "68.219.152.220/32", "72.144.95.185/32", "74.163.169.106/32", "74.163.208.206/32", "74.179.200.126/32", "74.179.232.116/32", "74.242.194.48/32", "98.70.219.37/32" ] }, { "id": "google-agent", "name": "Google-Agent", "operator": { "name": "Google", "url": "https://www.google.com" }, "description": "The fetcher used by AI agents hosted on Google infrastructure to navigate the web and perform actions on behalf of a user who asked for them. Google publishes a dedicated IP range list for it and signs a subset of its requests with Web Bot Auth under the agent.bot.goog identity. As a user-triggered fetcher it generally ignores robots.txt rules.", "docs": [ "https://developers.google.com/crawling/docs/crawlers-fetchers/google-agent", "https://developers.google.com/crawling/docs/crawlers-fetchers/web-bot-auth" ], "category": "ai-assistant", "user_agents": { "patterns": [ "Google-Agent" ], "instances": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Google-Agent; +https://developers.google.com/crawling/docs/crawlers-fetchers/google-agent) Chrome/W.X.Y.Z Safari/537.36", "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; Google-Agent; +https://developers.google.com/crawling/docs/crawlers-fetchers/google-agent)" ] }, "behavior": { "respects_robots_txt": false }, "verification": [ { "type": "cidr_feed", "url": "https://developers.google.com/static/crawling/ipranges/user-triggered-agents.json", "format": "prefixes" }, { "type": "web_bot_auth", "signature_agent_url": "https://agent.bot.goog/.well-known/http-message-signatures-directory" } ], "tier": 1, "tier_label": "Fully verifiable", "first_seen": "2026-08-04T20:03:13.870Z", "resolved_cidrs": [ "136.121.16.0/24", "136.121.24.0/21", "136.121.40.0/21", "136.122.0.0/16", "2001:4860:c::/124", "2001:4860:c::10/124", "2001:4860:c::20/124", "2001:4860:c::30/124", "2001:4860:c::40/124", "2001:4860:c::50/124", "2001:4860:c::60/124", "2001:4860:c::70/124", "74.125.232.0/28", "74.125.232.112/28", "74.125.232.16/28", "74.125.232.32/28", "74.125.232.48/28", "74.125.232.64/28", "74.125.232.80/28", "74.125.232.96/28" ] }, { "id": "google-user-triggered-fetchers", "name": "Google user-triggered fetchers", "operator": { "name": "Google", "url": "https://www.google.com" }, "description": "Google tools and product features that fetch a specific page because an end user asked for it (Google Read Aloud, Site Verifier, Gemini Notebook, Chrome Web Store, Google Messages, Pinpoint, Publisher Center, and similar), rather than autonomous crawling for search indexing. They generally ignore robots.txt because a human requested the fetch.", "docs": [ "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers" ], "category": "ai-assistant", "user_agents": { "patterns": [ "Google-Read-Aloud", "Google-Site-Verification", "Google-NotebookLM", "Google-GeminiNotebook", "Google-CWS", "GoogleMessages", "Google-Pinpoint", "GoogleProducer" ], "instances": [ "Mozilla/5.0 (compatible; Google-Site-Verification/1.0)", "Google-Read-Aloud", "Google-NotebookLM", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36 (compatible; Google-GeminiNotebook; +https://developers.google.com/crawling/docs/crawlers-fetchers/google-gemininotebook)", "Mozilla/5.0 (compatible; Google-CWS)", "GoogleMessages", "Google-Pinpoint", "GoogleProducer; (+https://developers.google.com/search/docs/crawling-indexing/google-producer)" ] }, "behavior": { "respects_robots_txt": false }, "verification": [ { "type": "cidr_feed", "url": "https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers.json", "format": "prefixes" } ], "tier": 1, "tier_label": "Fully verifiable", "first_seen": "2026-07-08T19:13:49.571Z", "resolved_cidrs": [ "107.178.192.0/27", "107.178.192.128/27", "107.178.192.160/27", "107.178.192.192/27", "107.178.192.224/27", "107.178.192.32/27", "107.178.192.64/27", "107.178.192.96/27", "107.178.193.0/27", "107.178.193.128/27", "107.178.193.160/27", "107.178.193.192/27", "107.178.193.224/27", "107.178.193.32/27", "107.178.193.64/27", "107.178.193.96/27", "107.178.194.0/27", "107.178.194.128/27", "107.178.194.160/27", "107.178.194.192/27", "107.178.194.224/27", "107.178.194.32/27", "107.178.194.64/27", "107.178.194.96/27", "107.178.195.0/27", "107.178.195.128/27", "107.178.195.160/27", "107.178.195.192/27", "107.178.195.224/27", "107.178.195.32/27", "107.178.195.64/27", "107.178.195.96/27", "107.178.196.0/27", "107.178.196.128/27", "107.178.196.160/27", "107.178.196.192/27", "107.178.196.224/27", "107.178.196.32/27", "107.178.196.64/27", "107.178.196.96/27", "107.178.197.0/27", "107.178.197.128/27", "107.178.197.160/27", "107.178.197.192/27", "107.178.197.224/27", "107.178.197.32/27", "107.178.197.64/27", "107.178.197.96/27", "107.178.198.0/27", "107.178.198.128/27", "107.178.198.160/27", "107.178.198.192/27", "107.178.198.224/27", "107.178.198.32/27", "107.178.198.64/27", "107.178.198.96/27", "107.178.199.0/27", "107.178.199.128/27", "107.178.199.160/27", "107.178.199.192/27", "107.178.199.224/27", "107.178.199.32/27", "107.178.199.64/27", "107.178.199.96/27", "107.178.202.0/27", "107.178.202.128/27", "107.178.202.160/27", "107.178.202.192/27", "107.178.202.224/27", "107.178.202.32/27", "107.178.202.64/27", "107.178.202.96/27", "107.178.203.0/27", "107.178.203.128/27", "107.178.203.160/27", "107.178.203.192/27", "107.178.203.224/27", "107.178.203.32/27", "107.178.203.64/27", "107.178.224.0/27", "107.178.224.128/27", "107.178.224.160/27", "107.178.224.192/27", "107.178.224.224/27", "107.178.224.32/27", "107.178.224.64/27", "107.178.224.96/27", "107.178.225.0/27", "107.178.225.128/27", "107.178.225.160/27", "107.178.225.192/27", "107.178.225.224/27", "107.178.225.32/27", "107.178.225.64/27", "107.178.225.96/27", "2404:f340:10:4000::/64", "2404:f340:10:4001::/64", "2404:f340:10:4002::/64", "2404:f340:10:4003::/64", "2404:f340:10:4004::/64", "2404:f340:10:4005::/64", "2404:f340:10:4006::/64", "2404:f340:10:4007::/64", "2404:f340:4010:4000::/64", "2404:f340:4010:4001::/64", "2404:f340:4010:4002::/64", "2404:f340:4010:4003::/64", "2404:f340:4010:4004::/64", "2404:f340:4010:4005::/64", "2404:f340:4010:4006::/64", "2404:f340:4010:4007::/64", "2600:1900:0:100::/64", "2600:1900:0:101::/64", "2600:1900:0:102::/64", "2600:1900:0:103::/64", "2600:1900:0:104::/64", "2600:1900:0:105::/64", "2600:1900:0:106::/64", "2600:1900:0:107::/64", "2600:1900:0:108::/64", "2600:1900:0:109::/64", "2600:1900:0:10::/64", "2600:1900:0:10a::/64", "2600:1900:0:10b::/64", "2600:1900:0:10c::/64", "2600:1900:0:10d::/64", "2600:1900:0:10e::/64", "2600:1900:0:10f::/64", "2600:1900:0:110::/64", "2600:1900:0:111::/64", "2600:1900:0:112::/64", "2600:1900:0:113::/64", "2600:1900:0:114::/64", "2600:1900:0:115::/64", "2600:1900:0:116::/64", "2600:1900:0:117::/64", "2600:1900:0:118::/64", "2600:1900:0:119::/64", "2600:1900:0:11a::/64", "2600:1900:0:11b::/64", "2600:1900:0:11c::/64", "2600:1900:0:11d::/64", "2600:1900:0:11e::/64", "2600:1900:0:11f::/64", "2600:1900:0:120::/64", "2600:1900:0:121::/64", "2600:1900:0:122::/64", "2600:1900:0:123::/64", "2600:1900:0:130::/64", "2600:1900:0:131::/64", "2600:1900:0:132::/64", "2600:1900:0:133::/64", "2600:1900:0:138::/64", "2600:1900:0:139::/64", "2600:1900:0:13a::/64", "2600:1900:0:13b::/64", "2600:1900:0:140::/64", "2600:1900:0:141::/64", "2600:1900:0:142::/64", "2600:1900:0:143::/64", "2600:1900:0:144::/64", "2600:1900:0:145::/64", "2600:1900:0:146::/64", "2600:1900:0:150::/64", "2600:1900:0:151::/64", "2600:1900:0:152::/64", "2600:1900:0:153::/64", "2600:1900:0:154::/64", "2600:1900:0:155::/64", "2600:1900:0:156::/64", "2600:1900:0:157::/64", "2600:1900:0:158::/64", "2600:1900:0:159::/64", "2600:1900:0:15::/64", "2600:1900:0:15a::/64", "2600:1900:0:15b::/64", "2600:1900:0:15c::/64", "2600:1900:0:15d::/64", "2600:1900:0:160::/64", "2600:1900:0:161::/64", "2600:1900:0:162::/64", "2600:1900:0:163::/64", "2600:1900:0:164::/64", "2600:1900:0:165::/64", "2600:1900:0:166::/64", "2600:1900:0:167::/64", "2600:1900:0:168::/64", "2600:1900:0:169::/64", "2600:1900:0:16::/64", "2600:1900:0:16a::/64", "2600:1900:0:16b::/64", "2600:1900:0:16c::/64", "2600:1900:0:16d::/64", "2600:1900:0:16e::/64", "2600:1900:0:16f::/64", "2600:1900:0:170::/64", "2600:1900:0:171::/64", "2600:1900:0:172::/64", "2600:1900:0:173::/64", "2600:1900:0:17::/64", "2600:1900:0:180::/64", "2600:1900:0:181::/64", "2600:1900:0:182::/64", "2600:1900:0:183::/64", "2600:1900:0:184::/64", "2600:1900:0:185::/64", "2600:1900:0:186::/64", "2600:1900:0:187::/64", "2600:1900:0:188::/64", "2600:1900:0:189::/64", "2600:1900:0:18::/64", "2600:1900:0:18a::/64", "2600:1900:0:18b::/64", "2600:1900:0:18c::/64", "2600:1900:0:18d::/64", "2600:1900:0:18e::/64", "2600:1900:0:18f::/64", "2600:1900:0:190::/64", "2600:1900:0:191::/64", "2600:1900:0:192::/64", "2600:1900:0:193::/64", "2600:1900:0:194::/64", "2600:1900:0:195::/64", "2600:1900:0:196::/64", "2600:1900:0:1a0::/64", "2600:1900:0:1a1::/64", "2600:1900:0:1a2::/64", "2600:1900:0:1a3::/64", "2600:1900:0:1a::/64", "2600:1900:0:1b::/64", "2600:1900:0:1c0::/64", "2600:1900:0:1c1::/64", "2600:1900:0:1c2::/64", "2600:1900:0:1c3::/64", "2600:1900:0:1c4::/64", "2600:1900:0:1c5::/64", "2600:1900:0:1c6::/64", "2600:1900:0:1c::/64", "2600:1900:0:1d0::/64", "2600:1900:0:1d1::/64", "2600:1900:0:1e0::/64", "2600:1900:0:1e1::/64", "2600:1900:0:1e2::/64", "2600:1900:0:1e3::/64", "2600:1900:0:1e4::/64", "2600:1900:0:1f0::/64", "2600:1900:0:1f1::/64", "2600:1900:0:1f2::/64", "2600:1900:0:1f3::/64", "2600:1900:0:1f4::/64", "2600:1900:0:1f5::/64", "2600:1900:0:1f6::/64", "2600:1900:0:1f7::/64", "2600:1900:0:1f8::/64", "2600:1900:0:200::/64", "2600:1900:0:201::/64", "2600:1900:0:202::/64", "2600:1900:0:203::/64", "2600:1900:0:204::/64", "2600:1900:0:205::/64", "2600:1900:0:206::/64", "2600:1900:0:207::/64", "2600:1900:0:208::/64", "2600:1900:0:209::/64", "2600:1900:0:20::/64", "2600:1900:0:20a::/64", "2600:1900:0:20b::/64", "2600:1900:0:210::/64", "2600:1900:0:211::/64", "2600:1900:0:212::/64", "2600:1900:0:213::/64", "2600:1900:0:214::/64", "2600:1900:0:215::/64", "2600:1900:0:216::/64", "2600:1900:0:217::/64", "2600:1900:0:21::/64", "2600:1900:0:220::/64", "2600:1900:0:221::/64", "2600:1900:0:222::/64", "2600:1900:0:223::/64", "2600:1900:0:22::/64", "2600:1900:0:230::/64", "2600:1900:0:231::/64", "2600:1900:0:232::/64", "2600:1900:0:233::/64", "2600:1900:0:234::/64", "2600:1900:0:235::/64", "2600:1900:0:236::/64", "2600:1900:0:237::/64", "2600:1900:0:238::/64", "2600:1900:0:239::/64", "2600:1900:0:23::/64", "2600:1900:0:23a::/64", "2600:1900:0:23b::/64", "2600:1900:0:240::/64", "2600:1900:0:241::/64", "2600:1900:0:242::/64", "2600:1900:0:24::/64", "2600:1900:0:250::/64", "2600:1900:0:251::/64", "2600:1900:0:252::/64", "2600:1900:0:253::/64", "2600:1900:0:254::/64", "2600:1900:0:255::/64", "2600:1900:0:25::/64", "2600:1900:0:260::/64", "2600:1900:0:261::/64", "2600:1900:0:262::/64", "2600:1900:0:263::/64", "2600:1900:0:264::/64", "2600:1900:0:26::/64", "2600:1900:0:270::/64", "2600:1900:0:271::/64", "2600:1900:0:272::/64", "2600:1900:0:273::/64", "2600:1900:0:274::/64", "2600:1900:0:275::/64", "2600:1900:0:276::/64", "2600:1900:0:27::/64", "2600:1900:0:280::/64", "2600:1900:0:281::/64", "2600:1900:0:282::/64", "2600:1900:0:28::/64", "2600:1900:0:290::/64", "2600:1900:0:291::/64", "2600:1900:0:292::/64", "2600:1900:0:293::/64", "2600:1900:0:294::/64", "2600:1900:0:295::/64", "2600:1900:0:296::/64", "2600:1900:0:297::/64", "2600:1900:0:29::/64", "2600:1900:0:2a0::/64", "2600:1900:0:2a1::/64", "2600:1900:0:2a2::/64", "2600:1900:0:2a3::/64", "2600:1900:0:2a::/64", "2600:1900:0:2b0::/64", "2600:1900:0:2b1::/64", "2600:1900:0:2b2::/64", "2600:1900:0:2b3::/64", "2600:1900:0:2b::/64", "2600:1900:0:2c0::/64", "2600:1900:0:2c1::/64", "2600:1900:0:2c2::/64", "2600:1900:0:2c3::/64", "2600:1900:0:2c4::/64", "2600:1900:0:2c5::/64", "2600:1900:0:2c6::/64", "2600:1900:0:2d0::/64", "2600:1900:0:2d1::/64", "2600:1900:0:2d2::/64", "2600:1900:0:2d3::/64", "2600:1900:0:2d4::/64", "2600:1900:0:2d5::/64", "2600:1900:0:2d6::/64", "2600:1900:0:2d7::/64", "2600:1900:0:2d8::/64", "2600:1900:0:2d9::/64", "2600:1900:0:2da::/64", "2600:1900:0:2db::/64", "2600:1900:0:2dc::/64", "2600:1900:0:2dd::/64", "2600:1900:0:2de::/64", "2600:1900:0:2df::/64", "2600:1900:0:2e0::/64", "2600:1900:0:2e1::/64", "2600:1900:0:2e2::/64", "2600:1900:0:2f0::/64", "2600:1900:0:2f1::/64", "2600:1900:0:2f2::/64", "2600:1900:0:2f3::/64", "2600:1900:0:2f4::/64", "2600:1900:0:2f5::/64", "2600:1900:0:2f6::/64", "2600:1900:0:300::/64", "2600:1900:0:301::/64", "2600:1900:0:302::/64", "2600:1900:0:303::/64", "2600:1900:0:30::/64", "2600:1900:0:310::/64", "2600:1900:0:311::/64", "2600:1900:0:312::/64", "2600:1900:0:313::/64", "2600:1900:0:314::/64", "2600:1900:0:315::/64", "2600:1900:0:316::/64", "2600:1900:0:317::/64", "2600:1900:0:31::/64", "2600:1900:0:320::/64", "2600:1900:0:321::/64", "2600:1900:0:322::/64", "2600:1900:0:32::/64", "2600:1900:0:330::/64", "2600:1900:0:331::/64", "2600:1900:0:332::/64", "2600:1900:0:333::/64", "2600:1900:0:334::/64", "2600:1900:0:335::/64", "2600:1900:0:33::/64", "2600:1900:0:340::/64", "2600:1900:0:34::/64", "2600:1900:0:350::/64", "2600:1900:0:351::/64", "2600:1900:0:352::/64", "2600:1900:0:353::/64", "2600:1900:0:354::/64", "2600:1900:0:35::/64", "2600:1900:0:360::/64", "2600:1900:0:361::/64", "2600:1900:0:362::/64", "2600:1900:0:36::/64", "2600:1900:0:370::/64", "2600:1900:0:371::/64", "2600:1900:0:372::/64", "2600:1900:0:373::/64", "2600:1900:0:374::/64", "2600:1900:0:37::/64", "2600:1900:0:380::/64", "2600:1900:0:381::/64", "2600:1900:0:382::/64", "2600:1900:0:383::/64", "2600:1900:0:384::/64", "2600:1900:0:385::/64", "2600:1900:0:386::/64", "2600:1900:0:387::/64", "2600:1900:0:388::/64", "2600:1900:0:389::/64", "2600:1900:0:38::/64", "2600:1900:0:38a::/64", "2600:1900:0:38b::/64", "2600:1900:0:38c::/64", "2600:1900:0:38d::/64", "2600:1900:0:38e::/64", "2600:1900:0:390::/64", "2600:1900:0:391::/64", "2600:1900:0:392::/64", "2600:1900:0:393::/64", "2600:1900:0:39::/64", "2600:1900:0:3a0::/64", "2600:1900:0:3a1::/64", "2600:1900:0:3a2::/64", "2600:1900:0:3a::/64", "2600:1900:0:3b0::/64", "2600:1900:0:3b1::/64", "2600:1900:0:3b2::/64", "2600:1900:0:3b3::/64", "2600:1900:0:3b4::/64", "2600:1900:0:3b5::/64", "2600:1900:0:3b6::/64", "2600:1900:0:3b::/64", "2600:1900:0:3c0::/64", "2600:1900:0:3c1::/64", "2600:1900:0:3c3::/64", "2600:1900:0:3c4::/64", "2600:1900:0:3c::/64", "2600:1900:0:3d0::/64", "2600:1900:0:3d1::/64", "2600:1900:0:3d2::/64", "2600:1900:0:3d3::/64", "2600:1900:0:3d4::/64", "2600:1900:0:3d5::/64", "2600:1900:0:3d6::/64", "2600:1900:0:3d7::/64", "2600:1900:0:40::/64", "2600:1900:0:41::/64", "2600:1900:0:42::/64", "2600:1900:0:43::/64", "2600:1900:0:44::/64", "2600:1900:0:45::/64", "2600:1900:0:46::/64", "2600:1900:0:47::/64", "2600:1900:0:48::/64", "2600:1900:0:49::/64", "2600:1900:0:4a::/64", "2600:1900:0:4b::/64", "2600:1900:0:4c::/64", "2600:1900:0:4d::/64", "2600:1900:0:4e::/64", "2600:1900:0:4f::/64", "2600:1900:0:50::/64", "2600:1900:0:51::/64", "2600:1900:0:52::/64", "2600:1900:0:53::/64", "2600:1900:0:54::/64", "2600:1900:0:55::/64", "2600:1900:0:56::/64", "2600:1900:0:57::/64", "2600:1900:0:58::/64", "2600:1900:0:59::/64", "2600:1900:0:5a::/64", "2600:1900:0:5b::/64", "2600:1900:0:5c::/64", "2600:1900:0:5d::/64", "2600:1900:0:5e::/64", "2600:1900:0:5f::/64", "2600:1900:0:60::/64", "2600:1900:0:61::/64", "2600:1900:0:62::/64", "2600:1900:0:64::/64", "2600:1900:0:65::/64", "2600:1900:0:66::/64", "2600:1900:0:67::/64", "2600:1900:0:68::/64", "2600:1900:0:69::/64", "2600:1900:0:6a::/64", "2600:1900:0:6b::/64", "2600:1900:0:6c::/64", "2600:1900:0:6d::/64", "2600:1900:0:6e::/64", "2600:1900:0:70::/64", "2600:1900:0:71::/64", "2600:1900:0:72::/64", "2600:1900:0:73::/64", "2600:1900:0:74::/64", "2600:1900:0:75::/64", "2600:1900:0:76::/64", "2600:1900:0:77::/64", "2600:1900:0:78::/64", "2600:1900:0:79::/64", "2600:1900:0:7a::/64", "2600:1900:0:7b::/64", "2600:1900:0:7c::/64", "2600:1900:0:7d::/64", "2600:1900:0:7e::/64", "2600:1900:0:7f::/64", "2600:1900:0:80::/64", "2600:1900:0:81::/64", "2600:1900:0:82::/64", "2600:1900:0:83::/64", "2600:1900:0:84::/64", "2600:1900:0:85::/64", "2600:1900:0:86::/64", "2600:1900:0:87::/64", "2600:1900:0:88::/64", "2600:1900:0:89::/64", "2600:1900:0:8::/64", "2600:1900:0:8a::/64", "2600:1900:0:8b::/64", "2600:1900:0:90::/64", "2600:1900:0:91::/64", "2600:1900:0:92::/64", "2600:1900:0:93::/64", "2600:1900:0:94::/64", "2600:1900:0:95::/64", "2600:1900:0:96::/64", "2600:1900:0:97::/64", "2600:1900:0:98::/64", "2600:1900:0:99::/64", "2600:1900:0:9::/64", "2600:1900:0:a0::/64", "2600:1900:0:a1::/64", "2600:1900:0:a2::/64", "2600:1900:0:a3::/64", "2600:1900:0:a4::/64", "2600:1900:0:a5::/64", "2600:1900:0:a6::/64", "2600:1900:0:b0::/64", "2600:1900:0:b1::/64", "2600:1900:0:b2::/64", "2600:1900:0:b3::/64", "2600:1900:0:b4::/64", "2600:1900:0:b5::/64", "2600:1900:0:b6::/64", "2600:1900:0:b7::/64", "2600:1900:0:b8::/64", "2600:1900:0:b9::/64", "2600:1900:0:ba::/64", "2600:1900:0:bb::/64", "2600:1900:0:bc::/64", "2600:1900:0:bd::/64", "2600:1900:0:c0::/64", "2600:1900:0:c1::/64", "2600:1900:0:c2::/64", "2600:1900:0:c3::/64", "2600:1900:0:c4::/64", "2600:1900:0:c5::/64", "2600:1900:0:c6::/64", "2600:1900:0:c::/64", "2600:1900:0:d0::/64", "2600:1900:0:d1::/64", "2600:1900:0:d2::/64", "2600:1900:0:d3::/64", "2600:1900:0:d4::/64", "2600:1900:0:d5::/64", "2600:1900:0:d6::/64", "2600:1900:0:d7::/64", "2600:1900:0:d8::/64", "2600:1900:0:d9::/64", "2600:1900:0:da::/64", "2600:1900:0:db::/64", "2600:1900:0:dc::/64", "2600:1900:0:dd::/64", "2600:1900:0:de::/64", "2600:1900:0:df::/64", "2600:1900:0:e0::/64", "2600:1900:0:e1::/64", "2600:1900:0:e2::/64", "2600:1900:0:e3::/64", "2600:1900:0:e4::/64", "2600:1900:0:e5::/64", "2600:1900:0:e6::/64", "2600:1900:0:e7::/64", "2600:1900:0:e8::/64", "2600:1900:0:e9::/64", "2600:1900:0:ea::/64", "2600:1900:0:eb::/64", "2600:1900:0:ec::/64", "2600:1900:0:ed::/64", "2600:1900:0:ee::/64", "2600:1900:0:ef::/64", "2600:1900:0:f0::/64", "2600:1900:0:f1::/64", "2600:1900:0:f2::/64", "2600:1900:0:f3::/64", "2600:1900:0:f4::/64", "2600:1900:0:f5::/64", "2600:1900:0:f6::/64", "2600:1900:0:f7::/64", "2600:1900:0:f8::/64", "2600:1900:0:f9::/64", "2600:1900:0:f::/64", "2600:1900:0:fa::/64", "34.101.6.0/27", "34.101.6.128/27", "34.101.6.160/27", "34.101.6.192/27", "34.101.6.224/27", "34.101.6.32/27", "34.101.6.64/27", "34.101.6.96/27", "34.116.16.0/27", "34.116.16.128/27", "34.116.16.160/27", "34.116.16.32/27", "34.116.16.64/27", "34.116.16.96/27", "34.116.17.0/27", "34.116.17.128/27", "34.116.17.32/27", "34.116.17.64/27", "34.116.17.96/27", "34.116.18.0/27", "34.116.18.128/27", "34.116.18.160/27", "34.116.18.192/27", "34.116.18.32/27", "34.116.18.64/27", "34.116.18.96/27", "34.116.19.0/27", "34.116.19.128/27", "34.116.19.160/27", "34.116.19.192/27", "34.116.19.224/27", "34.116.19.32/27", "34.116.19.64/27", "34.116.19.96/27", "34.116.20.0/27", "34.116.20.32/27", "34.116.20.64/27", "34.116.21.0/27", "34.116.21.128/27", "34.116.21.160/27", "34.116.21.192/27", "34.116.21.224/27", "34.116.21.32/27", "34.116.21.64/27", "34.116.21.96/27", "34.116.22.0/27", "34.116.22.32/27", "34.116.22.64/27", "34.116.22.96/27", "34.116.23.0/27", "34.116.23.32/27", "34.116.23.64/27", "34.116.23.96/27", "34.116.28.0/27", "34.116.28.128/27", "34.116.28.160/27", "34.116.28.192/27", "34.116.28.224/27", "34.116.28.32/27", "34.116.28.64/27", "34.116.28.96/27", "34.116.29.0/27", "34.116.29.32/27", "34.116.29.64/27", "34.116.29.96/27", "34.116.30.0/27", "34.116.30.128/27", "34.116.30.160/27", "34.116.30.192/27", "34.116.30.224/27", "34.116.30.32/27", "34.116.30.64/27", "34.116.30.96/27", "34.116.31.0/27", "34.116.31.32/27", "34.116.31.64/27", "34.116.32.0/27", "34.116.32.128/27", "34.116.32.160/27", "34.116.32.192/27", "34.116.32.32/27", "34.116.32.64/27", "34.116.32.96/27", "34.116.33.0/27", "34.116.33.128/27", "34.116.33.160/27", "34.116.33.192/27", "34.116.33.224/27", "34.116.33.32/27", "34.116.33.64/27", "34.116.33.96/27", "34.116.34.0/27", "34.116.34.32/27", "34.116.34.64/27", "34.116.35.0/27", "34.116.35.128/27", "34.116.35.160/27", "34.116.35.192/27", "34.116.35.32/27", "34.116.35.64/27", "34.116.35.96/27", "34.116.36.0/27", "34.116.36.128/27", "34.116.36.160/27", "34.116.36.32/27", "34.116.36.64/27", "34.116.36.96/27", "34.116.37.0/27", "34.116.38.0/27", "34.116.38.32/27", "34.116.39.0/27", "34.116.39.128/27", "34.116.39.160/27", "34.116.39.192/27", "34.116.39.224/27", "34.116.39.32/27", "34.116.39.64/27", "34.116.39.96/27", "34.116.40.0/27", "34.116.40.128/27", "34.116.40.160/27", "34.116.40.192/27", "34.116.40.224/27", "34.116.40.32/27", "34.116.40.64/27", "34.116.40.96/27", "34.116.41.0/27", "34.116.41.128/27", "34.116.41.160/27", "34.116.41.192/27", "34.116.41.224/27", "34.116.41.32/27", "34.116.41.64/27", "34.116.41.96/27", "34.116.42.0/27", "34.116.42.128/27", "34.116.42.32/27", "34.116.42.64/27", "34.116.42.96/27", "34.116.43.0/27", "34.116.43.128/27", "34.116.43.160/27", "34.116.43.192/27", "34.116.43.224/27", "34.116.43.32/27", "34.116.43.64/27", "34.116.43.96/27", "34.127.128.0/27", "34.127.128.128/27", "34.127.128.32/27", "34.127.128.64/27", "34.127.128.96/27", "34.127.129.0/27", "34.127.129.128/27", "34.127.129.32/27", "34.127.129.64/27", "34.127.129.96/27", "34.127.130.0/27", "34.127.130.32/27", "34.127.130.64/27", "34.127.130.96/27", "34.127.131.0/27", "34.127.131.32/27", "34.127.131.64/27", "34.127.132.0/27", "34.127.132.128/27", "34.127.132.160/27", "34.127.132.192/27", "34.127.132.224/27", "34.127.132.32/27", "34.127.132.64/27", "34.127.132.96/27", "34.127.133.0/27", "34.127.133.128/27", "34.127.133.160/27", "34.127.133.192/27", "34.127.133.224/27", "34.127.133.32/27", "34.127.133.64/27", "34.127.133.96/27", "34.127.134.0/27", "34.127.134.32/27", "34.127.134.64/27", "34.127.136.0/27", "34.127.136.128/27", "34.127.136.160/27", "34.127.136.192/27", "34.127.136.224/27", "34.127.136.32/27", "34.127.136.64/27", "34.127.136.96/27", "34.127.137.0/27", "34.127.137.32/27", "34.127.137.64/27", "34.127.137.96/27", "34.127.139.0/27", "34.127.140.0/27", "34.127.140.128/27", "34.127.140.160/27", "34.127.140.192/27", "34.127.140.224/27", "34.127.140.32/27", "34.127.140.64/27", "34.127.140.96/27", "34.127.142.0/27", "34.127.142.128/27", "34.127.142.32/27", "34.127.142.96/27", "34.127.143.0/27", "34.127.143.32/27", "34.127.143.64/27", "34.64.6.0/27", "34.64.6.128/27", "34.64.6.160/27", "34.64.6.192/27", "34.64.6.224/27", "34.64.6.32/27", "34.64.6.64/27", "34.64.6.96/27", "34.98.136.0/27", "34.98.136.128/27", "34.98.136.160/27", "34.98.136.192/27", "34.98.136.224/27", "34.98.136.32/27", "34.98.136.64/27", "34.98.136.96/27", "34.98.137.0/27", "34.98.137.32/27", "34.98.137.64/27", "34.98.137.96/27", "34.98.138.0/27", "34.98.138.128/27", "34.98.138.32/27", "34.98.138.64/27", "34.98.138.96/27", "34.98.139.0/27", "34.98.139.128/27", "34.98.139.160/27", "34.98.139.192/27", "34.98.139.224/27", "34.98.139.32/27", "34.98.139.64/27", "34.98.139.96/27", "34.98.140.0/27", "34.98.140.128/27", "34.98.140.160/27", "34.98.140.192/27", "34.98.140.224/27", "34.98.140.32/27", "34.98.140.64/27", "34.98.140.96/27", "34.98.141.0/27", "34.98.141.128/27", "34.98.141.160/27", "34.98.141.192/27", "34.98.141.224/27", "34.98.141.32/27", "34.98.141.64/27", "34.98.141.96/27", "34.98.142.0/27", "34.98.142.32/27", "34.98.142.64/27", "34.98.142.96/27", "34.98.143.0/27", "34.98.143.128/27", "34.98.143.160/27", "34.98.143.192/27", "34.98.143.224/27", "34.98.143.32/27", "34.98.143.64/27", "34.98.143.96/27", "35.187.128.0/27", "35.187.128.128/27", "35.187.128.160/27", "35.187.128.192/27", "35.187.128.224/27", "35.187.128.32/27", "35.187.128.64/27", "35.187.128.96/27", "35.187.129.0/27", "35.187.129.128/27", "35.187.129.160/27", "35.187.129.192/27", "35.187.129.224/27", "35.187.129.32/27", "35.187.129.64/27", "35.187.129.96/27", "35.187.130.0/27", "35.187.130.128/27", "35.187.130.160/27", "35.187.130.192/27", "35.187.130.224/27", "35.187.130.32/27", "35.187.130.64/27", "35.187.130.96/27", "35.187.131.0/27", "35.187.131.128/27", "35.187.131.160/27", "35.187.131.192/27", "35.187.131.224/27", "35.187.131.32/27", "35.187.131.64/27", "35.187.131.96/27", "35.187.132.0/27", "35.187.132.128/27", "35.187.132.160/27", "35.187.132.192/27", "35.187.132.224/27", "35.187.132.32/27", "35.187.132.64/27", "35.187.132.96/27", "35.187.133.0/27", "35.187.133.128/27", "35.187.133.160/27", "35.187.133.192/27", "35.187.133.224/27", "35.187.133.32/27", "35.187.133.64/27", "35.187.133.96/27", "35.187.134.0/27", "35.187.134.128/27", "35.187.134.160/27", "35.187.134.192/27", "35.187.134.224/27", "35.187.134.32/27", "35.187.134.64/27", "35.187.134.96/27", "35.187.135.0/27", "35.187.135.128/27", "35.187.135.160/27", "35.187.135.192/27", "35.187.135.224/27", "35.187.135.32/27", "35.187.135.64/27", "35.187.135.96/27", "35.187.136.0/27", "35.187.136.128/27", "35.187.136.160/27", "35.187.136.192/27", "35.187.136.224/27", "35.187.136.32/27", "35.187.138.0/27", "35.187.138.128/27", "35.187.138.160/27", "35.187.138.192/27", "35.187.138.224/27", "35.187.138.32/27", "35.187.138.64/27", "35.187.138.96/27", "35.187.139.0/27", "35.187.139.128/27", "35.187.139.160/27", "35.187.139.192/27", "35.187.139.224/27", "35.187.139.32/27", "35.187.139.64/27", "35.187.139.96/27", "35.187.140.0/27", "35.187.140.128/27", "35.187.140.160/27", "35.187.140.192/27", "35.187.140.32/27", "35.187.140.64/27", "35.187.140.96/27", "35.187.141.0/27", "35.187.141.128/27", "35.187.141.160/27", "35.187.141.192/27", "35.187.141.224/27", "35.187.141.32/27", "35.187.141.64/27", "35.187.142.0/27", "35.187.142.32/27", "35.187.142.64/27", "35.187.142.96/27", "35.187.143.0/27", "35.187.143.128/27", "35.187.143.160/27", "35.187.143.192/27", "35.187.143.32/27", "35.187.143.64/27", "35.187.143.96/27", "35.243.16.0/27", "35.243.16.128/27", "35.243.16.32/27", "35.243.16.64/27", "35.243.16.96/27", "35.243.17.0/27", "35.243.17.128/27", "35.243.17.160/27", "35.243.17.192/27", "35.243.17.32/27", "35.243.17.64/27", "35.243.17.96/27", "35.243.18.0/27", "35.243.18.32/27", "35.243.18.64/27", "35.243.18.96/27", "35.243.19.0/27", "35.243.19.128/27", "35.243.19.160/27", "35.243.19.192/27", "35.243.19.32/27", "35.243.19.64/27", "35.243.19.96/27", "35.243.20.0/27", "35.243.20.128/27", "35.243.20.160/27", "35.243.20.192/27", "35.243.20.224/27", "35.243.20.32/27", "35.243.20.64/27", "35.243.20.96/27", "35.243.21.0/27", "35.243.21.32/27", "35.243.21.64/27", "35.243.21.96/27", "35.243.22.0/27", "35.243.22.128/27", "35.243.22.160/27", "35.243.22.192/27", "35.243.22.32/27", "35.243.22.64/27", "35.243.22.96/27", "35.243.23.0/27", "35.243.23.128/27", "35.243.23.160/27", "35.243.23.192/27", "35.243.23.224/27", "35.243.23.32/27", "35.243.23.64/27", "35.243.23.96/27" ] }, { "id": "kimi-user", "name": "Kimi-User", "operator": { "name": "Moonshot AI", "url": "https://www.moonshot.ai" }, "description": "Moonshot AI's user-triggered fetcher for Kimi, which retrieves a page when a person asks Kimi to summarise it or answers a question needing live web retrieval. Moonshot states it is not used for automated bulk crawling and that, because the actions are user-triggered, robots.txt rules may not directly apply.", "docs": [ "https://www.kimi.ai/policies/kimi-crawlers" ], "category": "ai-assistant", "user_agents": { "patterns": [ "Kimi-User/\\d" ], "instances": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; Kimi-User/1.0; +https://www.kimi.com/policies/kimi-crawlers" ] }, "behavior": { "respects_robots_txt": false }, "verification": [ { "type": "cidr_feed", "url": "https://www.kimi.ai/policies/kimi-user.json", "format": "prefixes" } ], "tier": 1, "tier_label": "Fully verifiable", "first_seen": "2026-08-23T04:44:25.122Z", "resolved_cidrs": [ "47.79.225.241/32", "47.79.230.254/32", "47.79.253.42/32", "47.79.253.91/32" ] }, { "id": "meta-externalfetcher", "name": "Meta External Fetcher", "operator": { "name": "Meta", "url": "https://www.meta.com" }, "description": "Meta's on-demand fetcher that retrieves a single link at a user's request to support agentic AI features (e.g. an AI assistant navigating a page a user asked about), rather than broad indexing. Because its fetches are requested by a user, Meta documents that this crawler may bypass robots.txt rules. Verified by ASN lookup (AS32934).", "docs": [ "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/#identify-3" ], "category": "ai-assistant", "user_agents": { "patterns": [ "meta-externalfetcher/\\d" ], "instances": [ "meta-externalfetcher/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)", "meta-externalfetcher/1.1" ] }, "behavior": { "respects_robots_txt": false, "robots_token": "meta-externalfetcher" }, "verification": [ { "type": "asn", "numbers": [ 32934 ] } ], "tier": 2, "tier_label": "Verifiable", "first_seen": "2026-07-08T19:13:49.571Z", "resolved_cidrs": [] }, { "id": "mistralai-user", "name": "MistralAI-User", "operator": { "name": "Mistral AI", "url": "https://mistral.ai" }, "description": "Mistral AI's user-triggered fetcher, used when a user of a Mistral product (e.g. Le Chat) asks it to visit a specific web page to answer a question.", "docs": [ "https://docs.mistral.ai/robots" ], "category": "ai-assistant", "user_agents": { "patterns": [ "MistralAI-User/\\d" ], "instances": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; MistralAI-User/1.0; +https://docs.mistral.ai/robots)" ] }, "behavior": { "respects_robots_txt": true, "robots_token": "MistralAI-User" }, "verification": [ { "type": "cidr_feed", "url": "https://mistral.ai/mistralai-user-ips.json", "format": "prefixes" } ], "tier": 1, "tier_label": "Fully verifiable", "first_seen": "2026-07-08T19:13:49.571Z", "resolved_cidrs": [ "20.240.160.1/32", "20.240.160.161/32", "20.240.194.83/32", "51.12.243.114/32" ] }, { "id": "perplexity-user", "name": "Perplexity-User", "operator": { "name": "Perplexity", "url": "https://www.perplexity.ai" }, "description": "Perplexity's user-triggered fetcher, used when a user's question requires visiting a specific web page to produce an accurate answer.", "docs": [ "https://docs.perplexity.ai/docs/resources/perplexity-crawlers" ], "category": "ai-assistant", "user_agents": { "patterns": [ "Perplexity-User/\\d" ], "instances": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user)" ] }, "behavior": { "respects_robots_txt": false, "robots_token": "Perplexity-User" }, "verification": [ { "type": "cidr_feed", "url": "https://www.perplexity.com/perplexity-user.json", "format": "prefixes" } ], "tier": 1, "tier_label": "Fully verifiable", "first_seen": "2026-07-08T19:13:49.571Z", "resolved_cidrs": [ "18.97.21.0/30", "18.97.43.80/29", "34.193.163.52/32", "44.208.221.197/32" ] }, { "id": "shap-user", "name": "Shap-User", "operator": { "name": "Parallel Web Systems", "url": "https://parallel.ai" }, "description": "Parallel's user-triggered fetcher. It identifies itself when Parallel accesses content on behalf of a user, giving content owners visibility into user-initiated requests. The operator documents that it is not used for automatic crawling and is intended to provide visibility rather than to be managed through robots.txt, which governs ShapBot instead.", "docs": [ "https://parallel.ai/parallel-web-systems-bots" ], "category": "ai-assistant", "user_agents": { "patterns": [ "Shap-User/\\d" ], "instances": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; Shap-User/0.1.0" ] }, "behavior": { "respects_robots_txt": false }, "verification": [], "tier": 3, "tier_label": "Listed only", "first_seen": "2026-08-23T13:20:46.873Z", "resolved_cidrs": [] } ] }