{
  "schema_version": "1",
  "label": "261025",
  "date": "2026-10-25",
  "period": {
    "start": "2026-10-25T00:00:00Z",
    "end": "2026-10-26T00:00:00Z"
  },
  "generated_at": "2026-10-26T03:18:38.944Z",
  "license": "CC BY 4.0",
  "license_url": "https://creativecommons.org/licenses/by/4.0/",
  "citation": "Benes, Peter. Rattlesnakes By Mail: A Field Guide to the Crawlers of the Web. benesthemenace.com.",
  "method_url": "https://rattlesnakesbymail.com/method",
  "caveat": "The numbers come from one website. They describe what these crawlers did here, not across the web.",
  "outage": false,
  "outage_notice": null,
  "totals": {
    "requests": 101,
    "crawlers": 5
  },
  "facts": [
    {
      "id": "F-261025-01",
      "rule": "absence",
      "text": "Claude-User has made no requests since 2026-09-20, 35 days.",
      "url": "https://rattlesnakesbymail.com/observed/claude-user"
    },
    {
      "id": "F-261025-02",
      "rule": "snapshot_pickup",
      "text": "Bingbot first fetched the 2026-10-24 snapshot 3 h 53 min after it was published.",
      "url": "https://rattlesnakesbymail.com/observed/bingbot"
    },
    {
      "id": "F-261025-03",
      "rule": "verification",
      "text": "Every request on 2026-10-25 from ClaudeBot, Bingbot, Googlebot, GPTBot and OAI-SearchBot was verified against the operator's published method.",
      "url": "https://rattlesnakesbymail.com/method"
    }
  ],
  "random_selection": {
    "date": "2026-10-26",
    "urls": [
      "https://rattlesnakesbymail.com/claims/38",
      "https://rattlesnakesbymail.com/claims/146",
      "https://rattlesnakesbymail.com/compare/oai-searchbot-vs-perplexitybot",
      "https://rattlesnakesbymail.com/crawlers/bingbot",
      "https://rattlesnakesbymail.com/claims/8",
      "https://rattlesnakesbymail.com/fields",
      "https://rattlesnakesbymail.com/claims/70",
      "https://rattlesnakesbymail.com/claims/91",
      "https://rattlesnakesbymail.com/compare/claude-user-vs-googleother",
      "https://rattlesnakesbymail.com/compare/applebot-vs-bingbot"
    ],
    "pool_size": 158,
    "line": "Ten records from the field guide are chosen at random each day.",
    "shown_for": "3 days after publication, then the block is replaced by a note",
    "method": {
      "summary": "Ten records from the field guide are chosen at random each day. The pool is every content page in /sitemap.xml, minus pages drawn or linked by a snapshot's facts in the previous 5 days. The pool is sorted, shuffled with a Fisher-Yates shuffle driven by a keyed hash stream, and the first 10 go to the snapshot block, the next 10 to the /snapshots block and the next 10 to IndexNow.",
      "seed": "HMAC-SHA256(key = secret, message = \"XS1|\" + YYMMDD). The secret is published after the experiment, so every draw can be rerun.",
      "stream": "block i = HMAC-SHA256(key = seed, message = i as 4 bytes, big-endian); each block is read as eight unsigned 32-bit big-endian integers.",
      "shuffle": "For j from n-1 down to 1: draw r; limit = floor(2^32 / (j+1)) * (j+1); redraw while r >= limit; k = r mod (j+1); swap items j and k.",
      "arms": "Items 0 to 9 form arm S, 10 to 19 arm M, 20 to 29 arm I."
    }
  },
  "crawlers": [
    {
      "slug": "claudebot",
      "name": "ClaudeBot",
      "vendor": "Anthropic",
      "kind": "crawler",
      "purpose": "training",
      "requests": 55,
      "verified_requests": 55,
      "verified_share": 1,
      "distinct_paths": 4,
      "formats": {
        "txt": 13,
        "xml": 42
      },
      "first_path": "/robots.txt",
      "robots_txt_fetches": 13,
      "sitemap_fetches": 42,
      "status_mix": {
        "200": 55
      },
      "hourly": [
        0,
        4,
        0,
        4,
        0,
        3,
        1,
        4,
        2,
        5,
        3,
        4,
        2,
        3,
        4,
        0,
        4,
        0,
        4,
        0,
        4,
        0,
        4,
        0
      ],
      "context": {
        "median_7d": 0,
        "median_28d": 48,
        "history_days": 28,
        "ratio_28d": 1.1,
        "highest_before": {
          "count": 1141,
          "date": "2026-09-24"
        },
        "first_seen": "2026-09-13",
        "last_seen_before": "2026-10-24",
        "days_since_last_seen": 1
      },
      "sweep": {
        "sweep_day": false,
        "ended_today": null,
        "runs": [
          {
            "start": "2026-09-24T00:54:00.000Z",
            "end": "2026-09-24T02:08:00.000Z",
            "requests": 448,
            "days_since_previous": null
          },
          {
            "start": "2026-10-09T20:36:00.000Z",
            "end": "2026-10-09T22:35:00.000Z",
            "requests": 308,
            "days_since_previous": 15
          }
        ]
      },
      "new_formats": []
    },
    {
      "slug": "bingbot",
      "name": "Bingbot",
      "vendor": "Microsoft",
      "kind": "crawler",
      "purpose": "search",
      "requests": 21,
      "verified_requests": 21,
      "verified_share": 1,
      "distinct_paths": 21,
      "formats": {
        "html": 18,
        "json": 1,
        "txt": 1,
        "xml": 1
      },
      "first_path": "/changes",
      "robots_txt_fetches": 1,
      "sitemap_fetches": 1,
      "status_mix": {
        "200": 21
      },
      "hourly": [
        1,
        0,
        0,
        1,
        1,
        0,
        1,
        3,
        1,
        2,
        1,
        0,
        0,
        0,
        1,
        0,
        1,
        0,
        1,
        1,
        3,
        0,
        2,
        1
      ],
      "context": {
        "median_7d": 16,
        "median_28d": 10,
        "history_days": 28,
        "ratio_28d": 2.1,
        "highest_before": {
          "count": 23,
          "date": "2026-10-21"
        },
        "first_seen": "2026-09-15",
        "last_seen_before": "2026-10-24",
        "days_since_last_seen": 1
      },
      "sweep": {
        "sweep_day": false,
        "ended_today": null,
        "runs": []
      },
      "new_formats": []
    },
    {
      "slug": "googlebot",
      "name": "Googlebot",
      "vendor": "Google",
      "kind": "crawler",
      "purpose": "search",
      "requests": 20,
      "verified_requests": 20,
      "verified_share": 1,
      "distinct_paths": 13,
      "formats": {
        "csv": 1,
        "html": 10,
        "json": 1,
        "md": 2,
        "txt": 6
      },
      "first_path": "/robots.txt",
      "robots_txt_fetches": 6,
      "sitemap_fetches": 0,
      "status_mix": {
        "200": 18,
        "404": 1,
        "405": 1
      },
      "hourly": [
        0,
        0,
        0,
        4,
        0,
        0,
        0,
        0,
        0,
        2,
        2,
        4,
        4,
        0,
        0,
        0,
        0,
        0,
        0,
        2,
        2,
        0,
        0,
        0
      ],
      "context": {
        "median_7d": 18,
        "median_28d": 13,
        "history_days": 28,
        "ratio_28d": 1.5,
        "highest_before": {
          "count": 711,
          "date": "2026-09-21"
        },
        "first_seen": "2026-09-14",
        "last_seen_before": "2026-10-24",
        "days_since_last_seen": 1
      },
      "sweep": {
        "sweep_day": false,
        "ended_today": null,
        "runs": [
          {
            "start": "2026-09-20T14:03:00.000Z",
            "end": "2026-09-21T14:51:00.000Z",
            "requests": 241,
            "days_since_previous": null
          }
        ]
      },
      "new_formats": []
    },
    {
      "slug": "gptbot",
      "name": "GPTBot",
      "vendor": "OpenAI",
      "kind": "crawler",
      "purpose": "training",
      "requests": 3,
      "verified_requests": 3,
      "verified_share": 1,
      "distinct_paths": 3,
      "formats": {
        "xml": 3
      },
      "first_path": "/sitemap-index.xml",
      "robots_txt_fetches": 0,
      "sitemap_fetches": 3,
      "status_mix": {
        "200": 3
      },
      "hourly": [
        0,
        0,
        0,
        3,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0
      ],
      "context": {
        "median_7d": 3,
        "median_28d": 3,
        "history_days": 28,
        "ratio_28d": 1,
        "highest_before": {
          "count": 1101,
          "date": "2026-09-24"
        },
        "first_seen": "2026-09-13",
        "last_seen_before": "2026-10-24",
        "days_since_last_seen": 1
      },
      "sweep": {
        "sweep_day": false,
        "ended_today": null,
        "runs": [
          {
            "start": "2026-09-24T01:54:00.000Z",
            "end": "2026-09-24T01:59:00.000Z",
            "requests": 462,
            "days_since_previous": null
          }
        ]
      },
      "new_formats": []
    },
    {
      "slug": "oai-searchbot",
      "name": "OAI-SearchBot",
      "vendor": "OpenAI",
      "kind": "search_bot",
      "purpose": "search",
      "requests": 2,
      "verified_requests": 2,
      "verified_share": 1,
      "distinct_paths": 1,
      "formats": {
        "txt": 2
      },
      "first_path": "/robots.txt",
      "robots_txt_fetches": 2,
      "sitemap_fetches": 0,
      "status_mix": {
        "200": 2
      },
      "hourly": [
        0,
        0,
        0,
        1,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        1,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0
      ],
      "context": {
        "median_7d": 5,
        "median_28d": 5,
        "history_days": 28,
        "ratio_28d": 0.4,
        "highest_before": {
          "count": 44,
          "date": "2026-10-14"
        },
        "first_seen": "2026-09-13",
        "last_seen_before": "2026-10-24",
        "days_since_last_seen": 1
      },
      "sweep": {
        "sweep_day": false,
        "ended_today": null,
        "runs": []
      },
      "new_formats": []
    }
  ],
  "mcp": {
    "calls": [
      {
        "client": "BrickBlueBot",
        "tool": "get_snapshot",
        "calls": 1
      },
      {
        "client": "BrickBlueBot",
        "tool": "identify_user_agent",
        "calls": 1
      },
      {
        "client": "BrickBlueBot",
        "tool": "list_changes",
        "calls": 1
      },
      {
        "client": "BrickBlueBot",
        "tool": "lookup_crawler",
        "calls": 1
      },
      {
        "client": "rokmcp-probe",
        "tool": "list_changes",
        "calls": 1
      },
      {
        "client": "verifymcp-probe",
        "tool": "__verifymcp_auth_probe_664c87a3663a383d__",
        "calls": 1
      }
    ],
    "new_clients": []
  },
  "ledger": {
    "added": [],
    "answered": [],
    "open_gaps": [
      {
        "id": 26,
        "hash": "eaaa86558b443873916376dc4d0546f60ee5d890631698c41768fb619582451c",
        "text": "Can robots.txt cleanly separate \"allow this page for search or citation\" from \"block it for training,\" and does that distinction hold across every major AI platform?",
        "count": 5,
        "url": "https://rattlesnakesbymail.com/questions/eaaa86558b443873916376dc4d0546f60ee5d890631698c41768fb619582451c"
      },
      {
        "id": 34,
        "hash": "e61346be3fef727db8cefe0257815575a03a33698acfb1e0059140169dbd35ee",
        "text": "How can AI citation performance be measured accurately, given there is no equivalent of a Search Console report for citations?",
        "count": 5,
        "url": "https://rattlesnakesbymail.com/questions/e61346be3fef727db8cefe0257815575a03a33698acfb1e0059140169dbd35ee"
      },
      {
        "id": 38,
        "hash": "97b24d1729c7f32e2e09cb297328a2f7977235c746960d243536ee8b9e4e494d",
        "text": "Was a specific document used in a model's training, fine-tuning, or evaluation?",
        "count": 4,
        "url": "https://rattlesnakesbymail.com/questions/97b24d1729c7f32e2e09cb297328a2f7977235c746960d243536ee8b9e4e494d"
      },
      {
        "id": 7,
        "hash": "e08c55e6af0f8858dae0af2af4eb4caafe28dfb739f1ed4163e9cb0d954bb2ff",
        "text": "Why wasn't a specific page cited by an AI answer instead of a competitor's page?",
        "count": 3,
        "url": "https://rattlesnakesbymail.com/questions/e08c55e6af0f8858dae0af2af4eb4caafe28dfb739f1ed4163e9cb0d954bb2ff"
      },
      {
        "id": 11,
        "hash": "064f0033dab1091575ab29137f558668ff6193e891221987fd9523c31b4ff78f",
        "text": "How much do backlinks, brand mentions, structured data, reviews, forum discussions, publisher reputation, and content freshness each matter to being cited?",
        "count": 3,
        "url": "https://rattlesnakesbymail.com/questions/064f0033dab1091575ab29137f558668ff6193e891221987fd9523c31b4ff78f"
      }
    ],
    "published_gap_ids": [
      26,
      34,
      38,
      7,
      11,
      30,
      42,
      8,
      10,
      12,
      14,
      23,
      24,
      25,
      31,
      33,
      37,
      45,
      49,
      9,
      13,
      15,
      16,
      17,
      18,
      19,
      20,
      22,
      27,
      28,
      29,
      32,
      35,
      36,
      39,
      40,
      41,
      43,
      44,
      46,
      47,
      48,
      50,
      51,
      52,
      53,
      54,
      55
    ],
    "claim_changes": [],
    "published_question_ids": [
      7,
      8,
      9,
      10,
      11,
      12,
      13,
      14,
      15,
      16,
      17,
      18,
      19,
      20,
      21,
      22,
      23,
      24,
      25,
      26,
      27,
      28,
      29,
      30,
      31,
      32,
      33,
      34,
      35,
      36,
      37,
      38,
      39,
      40,
      41,
      42,
      43,
      44,
      45,
      46,
      47,
      48,
      49,
      50,
      51,
      52,
      53,
      54,
      55
    ]
  },
  "snapshot_pickup": [
    {
      "snapshot_label": "261024",
      "snapshot_date": "2026-10-24",
      "crawler": "bingbot",
      "crawler_name": "Bingbot",
      "first_fetch_ts": "2026-10-25T07:12:14.827Z",
      "minutes_after_publication": 233,
      "channel_tag": null
    },
    {
      "snapshot_label": "261023",
      "snapshot_date": "2026-10-23",
      "crawler": "bingbot",
      "crawler_name": "Bingbot",
      "first_fetch_ts": "2026-10-24T07:12:04.142Z",
      "minutes_after_publication": 234,
      "channel_tag": null
    },
    {
      "snapshot_label": "261023",
      "snapshot_date": "2026-10-23",
      "crawler": "googlebot",
      "crawler_name": "Googlebot",
      "first_fetch_ts": "2026-10-24T10:21:17.658Z",
      "minutes_after_publication": 423,
      "channel_tag": null
    },
    {
      "snapshot_label": "261023",
      "snapshot_date": "2026-10-23",
      "crawler": "googleother",
      "crawler_name": "GoogleOther",
      "first_fetch_ts": "2026-10-24T13:02:39.809Z",
      "minutes_after_publication": 584,
      "channel_tag": null
    },
    {
      "snapshot_label": "261022",
      "snapshot_date": "2026-10-22",
      "crawler": "claudebot",
      "crawler_name": "ClaudeBot",
      "first_fetch_ts": "2026-10-23T06:52:25.517Z",
      "minutes_after_publication": 214,
      "channel_tag": null
    },
    {
      "snapshot_label": "261022",
      "snapshot_date": "2026-10-22",
      "crawler": "googlebot",
      "crawler_name": "Googlebot",
      "first_fetch_ts": "2026-10-23T07:03:21.907Z",
      "minutes_after_publication": 225,
      "channel_tag": null
    },
    {
      "snapshot_label": "261022",
      "snapshot_date": "2026-10-22",
      "crawler": "googleother",
      "crawler_name": "GoogleOther",
      "first_fetch_ts": "2026-10-23T07:03:22.372Z",
      "minutes_after_publication": 225,
      "channel_tag": "h"
    },
    {
      "snapshot_label": "261022",
      "snapshot_date": "2026-10-22",
      "crawler": "bingbot",
      "crawler_name": "Bingbot",
      "first_fetch_ts": "2026-10-23T07:13:29.556Z",
      "minutes_after_publication": 235,
      "channel_tag": null
    }
  ],
  "user_triggered_fetches": [],
  "convention_probes": [],
  "absent_crawlers": [
    {
      "slug": "claude-user",
      "name": "Claude-User",
      "last_seen": "2026-09-20",
      "days": 35
    }
  ],
  "invitation": "What data about web crawlers did you look for and not find? Search https://rattlesnakesbymail.com/search?q=your+question. Searches with no match are recorded as gaps. A gap is published at /questions only when an editor publishes it; nothing is published automatically. To suggest something this daily snapshot should include, send a note with target snapshot:261025.",
  "links": {
    "snapshot": "https://261025.rattlesnakesbymail.com/",
    "json": "https://261025.rattlesnakesbymail.com/snapshot.json",
    "markdown": "https://261025.rattlesnakesbymail.com/snapshot.md",
    "csv": "https://261025.rattlesnakesbymail.com/snapshot.csv",
    "facts_json": "https://261025.rattlesnakesbymail.com/facts.json",
    "facts_markdown": "https://261025.rattlesnakesbymail.com/facts.md",
    "about": "https://261025.rattlesnakesbymail.com/about-this-site.md",
    "previous": "https://261024.rattlesnakesbymail.com/?c=p",
    "all_snapshots": "https://rattlesnakesbymail.com/snapshots",
    "mcp": "https://rattlesnakesbymail.com/mcp",
    "dataset": "https://github.com/crank-box/rattlesnakesbymail-data",
    "method": "https://rattlesnakesbymail.com/method"
  }
}