{
 "availability_movement": {
  "ok_to_unreachable": 141,
  "ratio_to_verdict_changes": 10.2,
  "total": 469,
  "unreachable_to_ok": 328
 },
 "counts": {
  "ai_verdict_changed": 46,
  "bytes_changed": 415,
  "comment_only": 12,
  "cosmetic_only": 29,
  "directives_changed": 386,
  "directives_changed_no_ai_verdict_moved": 340,
  "domains_parseable_both_scans": 12849,
  "domains_parseable_scan_1": 13035,
  "domains_parseable_scan_2": 13227,
  "whitespace_only": 17
 },
 "denominator_note": "All rates use domains returning a parseable robots.txt in BOTH scans. Domains that became unreachable are excluded and reported separately.",
 "direction_of_ai_verdict_changes": {
  "less_restrictive": 10,
  "mixed": 2,
  "more_restrictive": 14,
  "naming_only": 20
 },
 "flips_by_crawler": {
  "Amazonbot": {
   "to_allowed": 5,
   "to_blocked": 8
  },
  "Applebot-Extended": {
   "to_allowed": 6,
   "to_blocked": 9
  },
  "Bingbot": {
   "to_allowed": 4,
   "to_blocked": 1
  },
  "Bytespider": {
   "to_allowed": 6,
   "to_blocked": 9
  },
  "CCBot": {
   "to_allowed": 6,
   "to_blocked": 10
  },
  "ChatGPT-User": {
   "to_allowed": 6,
   "to_blocked": 4
  },
  "Claude-SearchBot": {
   "to_allowed": 4,
   "to_blocked": 3
  },
  "Claude-User": {
   "to_allowed": 4,
   "to_blocked": 3
  },
  "ClaudeBot": {
   "to_allowed": 6,
   "to_blocked": 8
  },
  "GPTBot": {
   "to_allowed": 8,
   "to_blocked": 7
  },
  "Google-Extended": {
   "to_allowed": 8,
   "to_blocked": 7
  },
  "Googlebot": {
   "to_allowed": 3,
   "to_blocked": 1
  },
  "MistralAI-User": {
   "to_allowed": 2,
   "to_blocked": 4
  },
  "OAI-SearchBot": {
   "to_allowed": 4,
   "to_blocked": 3
  },
  "Perplexity-User": {
   "to_allowed": 4,
   "to_blocked": 3
  },
  "PerplexityBot": {
   "to_allowed": 5,
   "to_blocked": 3
  },
  "anthropic-ai": {
   "to_allowed": 7,
   "to_blocked": 6
  },
  "meta-externalagent": {
   "to_allowed": 10,
   "to_blocked": 8
  }
 },
 "frame": {
  "crawlers_checked": 18,
  "list": "Tranco 38LNL",
  "n_requested": 25000,
  "ranks": "1-25,000"
 },
 "license": "CC BY 4.0",
 "license_url": "https://creativecommons.org/licenses/by/4.0/",
 "method": "One robots.txt fetch per domain, HTTPS with HTTP fallback, declared UA (DriftwitnessResearch/1.0, https://driftwitness.com/research-bot), no page crawling. Verdicts computed with Protego (RFC 9309) against the site root.",
 "not_claimed": [
  "A general or annualised rate of robots.txt change. This is one 72-hour window.",
  "Any direction of travel on AI blocking. The counts are too small.",
  "Any reason for any change. Files were observed, not intentions.",
  "Anything about domains outside the top 25,000 of this Tranco list, or about domains that did not answer in both scans."
 ],
 "publisher": "Driftwitness Research",
 "rates_pct": {
  "ai_verdict_changed": 0.358,
  "bytes_changed": 3.23,
  "directive_changes_moving_no_ai_verdict": 88.083
 },
 "title": "robots.txt 72-hour change rate, top 25,000 domains, August 2026",
 "url": "https://driftwitness.com/research/how-often-robots-txt-changes",
 "window": {
  "after": "2026-08-29",
  "before": "2026-08-26",
  "finished_utc": "2026-08-29T14:40:20Z",
  "hours": 72
 }
}