{
 "title": "AI crawler blocking rates, top 25000 domains",
 "publisher": "Driftwitness Research",
 "url": "https://driftwitness.com/research/who-blocks-ai-crawlers",
 "license": "CC BY 4.0",
 "scan_date": "2026-08-26",
 "source_list": {
  "name": "Tranco",
  "id": "38LNL",
  "generated": "2026-08-24",
  "url": "https://tranco-list.eu/list/38LNL"
 },
 "method": "One /robots.txt fetch per domain; rules evaluated per RFC 9309. Percentages use the responding-domain denominator; domains with no robots.txt count as not blocking.",
 "population": {
  "attempted": 25000,
  "responding": 17945,
  "parseable_robots": 13035,
  "unreachable": 6875,
  "no_robots": 3772,
  "served_html": 1138
 },
 "ai_aware_domains": {
  "n": 3375,
  "pct_of_responding": 18.8
 },
 "blocking_by_crawler": {
  "GPTBot": {
   "explicit_block_n": 2065,
   "explicit_block_pct_of_responding": 11.5,
   "any_block_pct_of_responding": 14.3,
   "named_by_n": 2727,
   "pct_of_namings_that_block_root": 75.7
  },
  "CCBot": {
   "explicit_block_n": 2078,
   "explicit_block_pct_of_responding": 11.6,
   "any_block_pct_of_responding": 14.5,
   "named_by_n": 2381,
   "pct_of_namings_that_block_root": 87.3
  },
  "Google-Extended": {
   "explicit_block_n": 1695,
   "explicit_block_pct_of_responding": 9.4,
   "any_block_pct_of_responding": 12.2,
   "named_by_n": 2217,
   "pct_of_namings_that_block_root": 76.5
  },
  "Applebot-Extended": {
   "explicit_block_n": 1615,
   "explicit_block_pct_of_responding": 9.0,
   "any_block_pct_of_responding": 11.9,
   "named_by_n": 1843,
   "pct_of_namings_that_block_root": 87.6
  },
  "meta-externalagent": {
   "explicit_block_n": 1655,
   "explicit_block_pct_of_responding": 9.2,
   "any_block_pct_of_responding": 12.2,
   "named_by_n": 1898,
   "pct_of_namings_that_block_root": 87.2
  },
  "Bytespider": {
   "explicit_block_n": 1984,
   "explicit_block_pct_of_responding": 11.1,
   "any_block_pct_of_responding": 14.0,
   "named_by_n": 2156,
   "pct_of_namings_that_block_root": 92.0
  },
  "Amazonbot": {
   "explicit_block_n": 1665,
   "explicit_block_pct_of_responding": 9.3,
   "any_block_pct_of_responding": 12.2,
   "named_by_n": 1931,
   "pct_of_namings_that_block_root": 86.2
  },
  "ClaudeBot": {
   "explicit_block_n": 1869,
   "explicit_block_pct_of_responding": 10.4,
   "any_block_pct_of_responding": 13.3,
   "named_by_n": 2415,
   "pct_of_namings_that_block_root": 77.4
  },
  "anthropic-ai": {
   "explicit_block_n": 784,
   "explicit_block_pct_of_responding": 4.4,
   "any_block_pct_of_responding": 7.3,
   "named_by_n": 1022,
   "pct_of_namings_that_block_root": 76.7
  },
  "Googlebot": {
   "explicit_block_n": 26,
   "explicit_block_pct_of_responding": 0.1,
   "any_block_pct_of_responding": 2.2,
   "named_by_n": 1048,
   "pct_of_namings_that_block_root": 2.5
  },
  "Bingbot": {
   "explicit_block_n": 98,
   "explicit_block_pct_of_responding": 0.5,
   "any_block_pct_of_responding": 2.9,
   "named_by_n": 854,
   "pct_of_namings_that_block_root": 11.5
  },
  "OAI-SearchBot": {
   "explicit_block_n": 402,
   "explicit_block_pct_of_responding": 2.2,
   "any_block_pct_of_responding": 5.0,
   "named_by_n": 965,
   "pct_of_namings_that_block_root": 41.7
  },
  "PerplexityBot": {
   "explicit_block_n": 724,
   "explicit_block_pct_of_responding": 4.0,
   "any_block_pct_of_responding": 6.8,
   "named_by_n": 1326,
   "pct_of_namings_that_block_root": 54.6
  },
  "Claude-SearchBot": {
   "explicit_block_n": 316,
   "explicit_block_pct_of_responding": 1.8,
   "any_block_pct_of_responding": 4.7,
   "named_by_n": 557,
   "pct_of_namings_that_block_root": 56.7
  },
  "ChatGPT-User": {
   "explicit_block_n": 656,
   "explicit_block_pct_of_responding": 3.7,
   "any_block_pct_of_responding": 6.4,
   "named_by_n": 1272,
   "pct_of_namings_that_block_root": 51.6
  },
  "Perplexity-User": {
   "explicit_block_n": 337,
   "explicit_block_pct_of_responding": 1.9,
   "any_block_pct_of_responding": 4.8,
   "named_by_n": 559,
   "pct_of_namings_that_block_root": 60.3
  },
  "Claude-User": {
   "explicit_block_n": 306,
   "explicit_block_pct_of_responding": 1.7,
   "any_block_pct_of_responding": 4.6,
   "named_by_n": 547,
   "pct_of_namings_that_block_root": 55.9
  },
  "MistralAI-User": {
   "explicit_block_n": 268,
   "explicit_block_pct_of_responding": 1.5,
   "any_block_pct_of_responding": 4.6,
   "named_by_n": 343,
   "pct_of_namings_that_block_root": 78.1
  }
 },
 "training_vs_search": {
  "GPTBot": {
   "search_bot": "OAI-SearchBot",
   "loose_n": 1669,
   "reverse_n": 11,
   "both_n": 892,
   "by_name_n": 1655,
   "strict_n": 103,
   "silent_n": 1552,
   "strict_pct": 6.2,
   "silent_pct": 93.8,
   "silent_in_ten": 9.4,
   "examples": [
    "yahoo.com",
    "snapchat.com",
    "forbes.com",
    "ebay.com",
    "tripadvisor.com",
    "globalsign.com",
    "ted.com",
    "academia.edu",
    "ebay.co.uk",
    "ouest-france.fr"
   ]
  },
  "ClaudeBot": {
   "search_bot": "Claude-SearchBot",
   "loose_n": 1552,
   "reverse_n": 5,
   "both_n": 832,
   "by_name_n": 1538,
   "strict_n": 64,
   "silent_n": 1474,
   "strict_pct": 4.2,
   "silent_pct": 95.8,
   "silent_in_ten": 9.6,
   "examples": [
    "yahoo.com",
    "snapchat.com",
    "forbes.com",
    "ebay.com",
    "globalsign.com",
    "ted.com",
    "ebay.co.uk",
    "pexels.com",
    "vnexpress.net",
    "fmkorea.com"
   ]
  }
 },
 "posture": {
  "blockers_n": 3125,
  "blockers_pct": 17.4,
  "all_n": 734,
  "all_pct_of_blockers": 23.5,
  "wildcard_only_n": 476,
  "n_training_bots_tested": 9,
  "eight_or_nine_n": 1872,
  "eight_or_nine_pct": 59.9,
  "distribution": {
   "1": 369,
   "2": 159,
   "3": 180,
   "4": 103,
   "5": 101,
   "6": 149,
   "7": 192,
   "8": 1138,
   "9": 734
  }
 },
 "by_rank_band": [
  {
   "band": "1\u20131,000",
   "n": 669,
   "aware_pct": 24.7,
   "GPTBot": 12.3,
   "ClaudeBot": 12.1,
   "PerplexityBot": 9.6,
   "OAI-SearchBot": 4.8
  },
  {
   "band": "1,001\u20135,000",
   "n": 2801,
   "aware_pct": 20.2,
   "GPTBot": 11.9,
   "ClaudeBot": 10.6,
   "PerplexityBot": 6.0,
   "OAI-SearchBot": 3.0
  },
  {
   "band": "5,001\u201310,000",
   "n": 3746,
   "aware_pct": 19.3,
   "GPTBot": 11.9,
   "ClaudeBot": 10.9,
   "PerplexityBot": 4.9,
   "OAI-SearchBot": 3.0
  },
  {
   "band": "10,001\u201325,000",
   "n": 10729,
   "aware_pct": 17.9,
   "GPTBot": 11.2,
   "ClaudeBot": 10.1,
   "PerplexityBot": 2.9,
   "OAI-SearchBot": 1.6
  }
 ]
}