{
  "name": "Content Signals adoption and provenance, top 25,000 domains, August 2026",
  "version": "1.0.0",
  "published": "2026-08-26",
  "license": "CC BY 4.0 (https://creativecommons.org/licenses/by/4.0/)",
  "attribution": "Driftwitness Research, https://driftwitness.com/research/",
  "article": "https://driftwitness.com/research/who-chose-to-block-ai-training",
  "contact": "research@driftwitness.com",
  "method": {
    "source_list": "Tranco 38LNL, generated 2026-08-24",
    "domains_attempted": 25000,
    "domains_responding": 17945,
    "domains_with_parseable_robots_txt": 13035,
    "denominator_note": "Percentages use domains that actually served a parseable robots.txt. That is the conservative denominator: a site with no robots.txt cannot carry a content signal, so including it would deflate every rate.",
    "user_agent": "Mozilla/5.0 (compatible; DriftwitnessResearch/1.0; +https://driftwitness.com/research-bot)",
    "collection_window": "2026-08-24 to 2026-08-26",
    "analysis_note": "Derived from archived response bodies. No domain was re-fetched for this analysis except the ten named in the article, which were verified individually on 2026-08-26.",
    "provenance_test": {
      "machine_marker_1": "presence of the Content Signals policy preamble (matches 'DIRECTIVE 2019/790' or 'As a condition of accessing this website, you agree to abide')",
      "machine_marker_2": "Content-Signal value byte-identical to 'search=yes,ai-train=no,use=reference'",
      "limitation": "A site owner who enabled the managed feature and agreed with the default is indistinguishable in the file from one who did not read it. This measures provenance, not intent."
    }
  },
  "adoption": {
    "signal_carriers": 1104,
    "signal_carriers_pct_of_bodies": 8.5,
    "cloudflare_block_present": 1059,
    "cloudflare_block_pct_of_bodies": 8.1
  },
  "provenance_split": {
    "machine_boilerplate_with_signal": 912,
    "machine_boilerplate_without_signal": 147,
    "signal_without_machine_boilerplate": 192,
    "byte_identical_default": 870,
    "default_pct_of_all_signal_carriers": 78.8,
    "default_pct_of_machine_managed": 95.4
  },
  "ai_train_preference": {
    "machine_managed": {
      "n": 912,
      "no": 910,
      "yes": 2,
      "no_pct": 99.8,
      "yes_pct": 0.2
    },
    "hand_authored": {
      "n": 192,
      "no": 108,
      "yes": 82,
      "no_pct": 56.2,
      "yes_pct": 42.7
    }
  },
  "by_rank_band": {
    "1-1k": {
      "domains_with_robots_txt": 480,
      "signal_carriers": 20,
      "pct": 4.2
    },
    "1k-10k": {
      "domains_with_robots_txt": 4870,
      "signal_carriers": 356,
      "pct": 7.3
    },
    "10k-100k": {
      "domains_with_robots_txt": 7685,
      "signal_carriers": 728,
      "pct": 9.5
    }
  },
  "author_written_value_distribution": [
    {
      "value": "ai-train=yes, search=yes, ai-input=yes",
      "count": 46
    },
    {
      "value": "ai-train=no, search=yes, ai-input=yes",
      "count": 38
    },
    {
      "value": "search=yes, ai-input=yes, ai-train=no",
      "count": 30
    },
    {
      "value": "search=yes, ai-input=yes, ai-train=yes",
      "count": 29
    },
    {
      "value": "ai-train=no, search=yes, ai-input=no",
      "count": 16
    },
    {
      "value": "search=yes,ai-train=no",
      "count": 8
    },
    {
      "value": "ai-train=yes, ai-input=yes, search=yes",
      "count": 3
    },
    {
      "value": "search=yes,ai-input=no,ai-train=no",
      "count": 3
    },
    {
      "value": "search=yes, ai-train=no, ai-input=yes",
      "count": 2
    },
    {
      "value": "search=yes, ai-train=yes, ai-input=yes",
      "count": 2
    }
  ],
  "limitations": [
    "The Tranco list skews toward large and Western sites; this is not a random sample of the web.",
    "Boilerplate detection is a heuristic in both directions: hand-copied preambles count as machine-assisted, and edited managed files count as author-written.",
    "Only the first Content-Signal line in each file is read; per-user-agent signal scoping is not modelled.",
    "Single point in time. Nothing here measures change over time."
  ]
}