{ "generated_at": "2026-10-16T09:25:04+00:00", "reporting_code": "5a4b2d4446471bb8", "what": "Share of sites whose robots.txt takes a stance on AI agents, by Common Crawl epoch.", "method": "Re-derived from Common Crawl captures; each point is a dated observation in an append-only signed chain, not a snapshot taken today.", "vantage": "bulk:commoncrawl", "min_n": 30, "epochs": 21, "series": [ { "epoch": "CC-MAIN-2019-04", "median_observed": 1548324648.0, "n": 69, "stance": 0, "rate": 0.0, "ci": [ 0.0, 0.05273909414400562 ], "blocking": 0, "naming": 0, "undersampled": false }, { "epoch": "CC-MAIN-2020-05", "median_observed": 1580318192.0, "n": 71, "stance": 0, "rate": 0.0, "ci": [ 0.0, 0.05132974174790491 ], "blocking": 0, "naming": 0, "undersampled": false }, { "epoch": "CC-MAIN-2021-04", "median_observed": 1611829781.0, "n": 74, "stance": 0, "rate": 0.0, "ci": [ 0.0, 0.049351503566216516 ], "blocking": 0, "naming": 0, "undersampled": false }, { "epoch": "CC-MAIN-2022-05", "median_observed": 1643437627.0, "n": 77, "stance": 0, "rate": 0.0, "ci": [ 3.469446951953614e-18, 0.047520088667220836 ], "blocking": 0, "naming": 0, "undersampled": false }, { "epoch": "CC-MAIN-2023-06", "median_observed": 1675913145.0, "n": 78, "stance": 0, "rate": 0.0, "ci": [ 0.0, 0.04693945377411976 ], "blocking": 0, "naming": 0, "undersampled": false }, { "epoch": "CC-MAIN-2023-23", "median_observed": 1686440014.0, "n": 78, "stance": 0, "rate": 0.0, "ci": [ 0.0, 0.04693945377411976 ], "blocking": 0, "naming": 0, "undersampled": false }, { "epoch": "CC-MAIN-2023-40", "median_observed": 1696469007.0, "n": 78, "stance": 3, "rate": 0.038461538461538464, "ci": [ 0.013165586780815211, 0.10708621670298764 ], "blocking": 3, "naming": 0, "undersampled": false }, { "epoch": "CC-MAIN-2024-10", "median_observed": 1709642508.0, "n": 81, "stance": 7, "rate": 0.08641975308641975, "ci": [ 0.04249204303723253, 0.1678010207427999 ], "blocking": 7, "naming": 0, "undersampled": false }, { "epoch": "CC-MAIN-2025-05", "median_observed": 1737893203.0, "n": 83, "stance": 4, "rate": 0.04819277108433735, "ci": [ 0.01889830967407153, 0.11746030186694112 ], "blocking": 3, "naming": 1, "undersampled": false }, { "epoch": "CC-MAIN-2025-26", "median_observed": 1750834547.0, "n": 81, "stance": 5, "rate": 0.06172839506172839, "ci": [ 0.026652119089554074, 0.13649426193107733 ], "blocking": 4, "naming": 1, "undersampled": false }, { "epoch": "CC-MAIN-2025-38", "median_observed": 1758174464.0, "n": 79, "stance": 3, "rate": 0.0379746835443038, "ci": [ 0.012997851302069409, 0.10580236501437063 ], "blocking": 3, "naming": 0, "undersampled": false }, { "epoch": "CC-MAIN-2025-51", "median_observed": 1765955838.0, "n": 79, "stance": 5, "rate": 0.06329113924050633, "ci": [ 0.0273344618147294, 0.13975067415560433 ], "blocking": 5, "naming": 0, "undersampled": false }, { "epoch": "CC-MAIN-2026-04", "median_observed": 1769339256.0, "n": 81, "stance": 5, "rate": 0.06172839506172839, "ci": [ 0.026652119089554074, 0.13649426193107733 ], "blocking": 5, "naming": 0, "undersampled": false }, { "epoch": "CC-MAIN-2026-08", "median_observed": 1771484947.0, "n": 81, "stance": 4, "rate": 0.04938271604938271, "ci": [ 0.01936899210168111, 0.12020404753924974 ], "blocking": 4, "naming": 0, "undersampled": false }, { "epoch": "CC-MAIN-2026-12", "median_observed": 1773752155.0, "n": 82, "stance": 6, "rate": 0.07317073170731707, "ci": [ 0.03396431451476119, 0.15058024197067243 ], "blocking": 5, "naming": 1, "undersampled": false }, { "epoch": "CC-MAIN-2026-17", "median_observed": 1776904651.0, "n": 83, "stance": 7, "rate": 0.08433734939759036, "ci": [ 0.04145280918254788, 0.16399711343518372 ], "blocking": 5, "naming": 2, "undersampled": false }, { "epoch": "CC-MAIN-2026-21", "median_observed": 1779379802.0, "n": 82, "stance": 8, "rate": 0.0975609756097561, "ci": [ 0.05026808709378501, 0.18087392330676672 ], "blocking": 4, "naming": 4, "undersampled": false }, { "epoch": "CC-MAIN-2026-25", "median_observed": 1781784301.0, "n": 84, "stance": 11, "rate": 0.13095238095238096, "ci": [ 0.07472144544944861, 0.21946263134332386 ], "blocking": 6, "naming": 5, "undersampled": false }, { "epoch": "CC-MAIN-2026-30", "median_observed": 1784747631.0, "n": 84, "stance": 11, "rate": 0.13095238095238096, "ci": [ 0.07472144544944861, 0.21946263134332386 ], "blocking": 6, "naming": 5, "undersampled": false }, { "epoch": "CC-MAIN-2026-34", "median_observed": 1787171791.0, "n": 82, "stance": 13, "rate": 0.15853658536585366, "ci": [ 0.09505500489170982, 0.2525806402966371 ], "blocking": 6, "naming": 7, "undersampled": false }, { "epoch": "CC-MAIN-2026-39", "median_observed": 1789594489.0, "n": 85, "stance": 12, "rate": 0.1411764705882353, "ci": [ 0.08264151586335423, 0.230743204808043 ], "blocking": 7, "naming": 5, "undersampled": false } ], "withheld": [ { "epoch": "CC-MAIN-2024-33", "median_observed": 1723778006.0, "n": 6, "stance": 0, "rate": 0.0, "ci": [ 0.0, 0.3903430336530645 ], "blocking": 0, "naming": 0, "undersampled": true, "withheld_because": "n=6 < MIN_N=30" } ], "withheld_note": "Epochs resolving fewer than MIN_N sites. Listed so the withholding is auditable, and kept OUT of `series` so that plotting the series gives the corrected view. One of these (a 6-site smoke test) measurably distorted the slope before it was withheld.", "live_cross_section": { "epoch": "live", "median_observed": 1792139039.9480016, "n": 555, "stance": 111, "rate": 0.2, "ci": [ 0.16883384270198373, 0.23529068918683774 ], "blocking": 24, "naming": 87, "undersampled": false, "as_of": "2026-10-16T09:25:04Z", "median_observed_iso": "2026-10-16T08:23:59Z", "freshness": { "rule": "a host is in the cross-section only if it was observed within 7 days of 2026-10-16T09:25:04Z", "fresh_within_days": 7.0, "in_rotation": { "n": 555, "oldest_observed": "2026-10-14T12:31:41Z", "newest_observed": "2026-10-16T08:56:34Z" }, "excluded_stale": { "n": 1335, "rate": 0.26666666666666666, "stance": 356, "blocking": 51, "naming": 305, "oldest_observed": "2026-09-20T11:50:27Z", "newest_observed": "2026-10-08T05:50:53Z", "why": "last observed outside the window. Their verdicts are real and remain in the chain; they are simply not evidence about today, so they are not averaged into a rate presented as current.", "not_a_trend": "This rate and the cross-section rate are over DISJOINT host sets, not the same hosts at two dates, so their difference is a sampling difference and must not be read as change over time. The series above is the only object in this document that measures change." }, "pooled_if_combined": { "n": 1890, "rate": 0.24708994708994708, "note": "what this field reported before 2026-10-14: every host ever probed live, whenever probed, in one rate. Kept here so the correction is auditable and its size visible, NOT for quoting." } } }, "live_cross_section_note": "NOT a point in the series above: different vantage (Portugal, live fetch) and a different population including machine-facing hosts. Reported so the difference is visible, not so the two can be concatenated. RESTRICTED to hosts observed within 7 days \u2014 a host enters the live cell on first probe and never leaves it, so the unrestricted figure averaged a weeks-old photograph with today and called the result current. The excluded pool is reported inside `freshness` with its own rate, and the previously published pooled figure is kept there too. Read `freshness.excluded_stale.not_a_trend` before comparing the two rates.", "unclassified_rows": [ { "epoch": "CC-MAIN-2024-33", "median_observed": 1723778006.0, "n": 6, "stance": 0, "rate": 0.0, "ci": [ 0.0, 0.3903430336530645 ], "blocking": 0, "naming": 0, "undersampled": true } ], "caveats": [ "Research demonstrator. Read the caveats before quoting a number.", "POPULATION: the reconstructed series resolves against the brochure corpus (~85 sites with usable Common Crawl history), NOT the machine-facing services this project now targets. The MCP registry barely existed before 2025, so that population has no reconstructable past.", "VANTAGE: held constant at 'bulk:commoncrawl'. Common Crawl fetches from us-east-1 and we fetch from Portugal, so a series mixing them would mix WHERE with WHEN. The live cross-section is a DIFFERENT vantage and a DIFFERENT population and is reported separately, never as the next point in the series.", "PRIOR ART: the fall-2023 onset is not our discovery. It is independently reported in 'Consent in Crisis: The Rapid Decline of the AI Data Commons' (arXiv 2407.14933).", "COMPOSITION: the 2026 rise is driven by sites NAMING AI agents rather than BLOCKING them. Blocking is close to saturated, so read 'blocking' and 'naming' rather than the rate alone.", "WITHHELD, NOT DROPPED: epochs resolving fewer than MIN_N=30 sites are listed with undersampled=true rather than silently removed.", "A rate is absent/(absent+present) \u2014 an evaluated denominator. Hosts that never answered are excluded, not counted as present.", "AS OF: the live cross-section counts only hosts observed in the last 7 days. A host that has not been re-probed is not evidence about today, so it is withheld and listed under `freshness`, never averaged in. The reconstructed series is unaffected: its epochs are dated by construction." ], "contact": "https://atlas.geosensor.tech/" }