{
  "version": "1.0.0",
  "generatedAt": "2026-08-14T21:31:19.036Z",
  "study": {
    "id": 1,
    "slug": "01-ai-crawlers",
    "state": "review_pending"
  },
  "window": {
    "start": "2026-07-15T21:20:49.000Z",
    "plannedEnd": "2026-08-14T21:20:49.000Z",
    "observedThrough": "2026-08-14T21:13:04.000Z",
    "elapsedDays": 29.99,
    "targetDays": 30,
    "progressPercent": 100
  },
  "totals": {
    "productionRequestsProcessed": 473211,
    "declaredCrawlerRequests": 123512,
    "verifiedRequests": 12425,
    "unverifiedRequests": 111087,
    "crawlerIdentities": 13
  },
  "crawlers": [
    {
      "crawler": "GPTBot",
      "operator": "OpenAI",
      "purpose": "model-development",
      "declaredRequests": 4886,
      "verifiedRequests": 4719,
      "unverifiedRequests": 167,
      "successfulResponses": 3993,
      "uniquePaths": 1641,
      "firstSeen": "2026-07-16T14:10:53.000Z",
      "lastSeen": "2026-08-14T00:00:45.000Z",
      "pageTypes": {
        "asset": 477,
        "other-page": 3469,
        "store": 126,
        "sitemap": 53,
        "home": 26,
        "journal-story-or-hub": 684,
        "services": 50,
        "vit-market": 1
      }
    },
    {
      "crawler": "Googlebot",
      "operator": "Google",
      "purpose": "traditional-search",
      "declaredRequests": 6765,
      "verifiedRequests": 3127,
      "unverifiedRequests": 3638,
      "successfulResponses": 5746,
      "uniquePaths": 1394,
      "firstSeen": "2026-07-15T21:25:57.000Z",
      "lastSeen": "2026-08-14T20:41:38.000Z",
      "pageTypes": {
        "robots": 771,
        "home": 54,
        "asset": 1540,
        "other-page": 2368,
        "journal-story-or-hub": 1796,
        "store": 146,
        "services": 43,
        "sitemap": 42,
        "vit-market": 5
      }
    },
    {
      "crawler": "OAI-SearchBot",
      "operator": "OpenAI",
      "purpose": "ai-search",
      "declaredRequests": 2325,
      "verifiedRequests": 2142,
      "unverifiedRequests": 183,
      "successfulResponses": 2109,
      "uniquePaths": 582,
      "firstSeen": "2026-07-15T22:46:27.000Z",
      "lastSeen": "2026-08-14T20:31:00.000Z",
      "pageTypes": {
        "robots": 151,
        "asset": 191,
        "other-page": 1175,
        "store": 148,
        "journal-story-or-hub": 445,
        "services": 84,
        "home": 39,
        "sitemap": 25,
        "vit-market": 67
      }
    },
    {
      "crawler": "bingbot",
      "operator": "Microsoft",
      "purpose": "traditional-search",
      "declaredRequests": 2132,
      "verifiedRequests": 1894,
      "unverifiedRequests": 238,
      "successfulResponses": 1381,
      "uniquePaths": 1057,
      "firstSeen": "2026-07-15T22:19:06.000Z",
      "lastSeen": "2026-08-14T20:52:41.000Z",
      "pageTypes": {
        "other-page": 1066,
        "journal-story-or-hub": 550,
        "services": 8,
        "asset": 297,
        "store": 18,
        "robots": 68,
        "home": 18,
        "sitemap": 105,
        "vit-market": 2
      }
    },
    {
      "crawler": "ChatGPT-User",
      "operator": "OpenAI",
      "purpose": "user-directed",
      "declaredRequests": 1162,
      "verifiedRequests": 543,
      "unverifiedRequests": 619,
      "successfulResponses": 858,
      "uniquePaths": 405,
      "firstSeen": "2026-07-15T23:48:07.000Z",
      "lastSeen": "2026-08-14T21:13:04.000Z",
      "pageTypes": {
        "sitemap": 10,
        "asset": 172,
        "other-page": 260,
        "home": 157,
        "services": 28,
        "journal-story-or-hub": 503,
        "store": 22,
        "robots": 1,
        "vit-market": 6,
        "research": 3
      }
    },
    {
      "crawler": "meta-externalagent",
      "operator": "Meta",
      "purpose": "model-development",
      "declaredRequests": 99669,
      "verifiedRequests": 0,
      "unverifiedRequests": 99669,
      "successfulResponses": 74514,
      "uniquePaths": 4397,
      "firstSeen": "2026-07-15T22:15:42.000Z",
      "lastSeen": "2026-08-14T18:20:56.000Z",
      "pageTypes": {
        "journal-story-or-hub": 17526,
        "other-page": 71773,
        "asset": 3266,
        "sitemap": 1104,
        "services": 1379,
        "store": 3024,
        "home": 758,
        "vit-market": 771,
        "research": 68
      }
    },
    {
      "crawler": "Amazonbot",
      "operator": "Amazon",
      "purpose": "model-development",
      "declaredRequests": 3193,
      "verifiedRequests": 0,
      "unverifiedRequests": 3193,
      "successfulResponses": 1836,
      "uniquePaths": 1805,
      "firstSeen": "2026-07-16T12:57:46.000Z",
      "lastSeen": "2026-08-14T18:14:12.000Z",
      "pageTypes": {
        "journal-story-or-hub": 586,
        "other-page": 1972,
        "asset": 543,
        "home": 25,
        "store": 45,
        "sitemap": 1,
        "services": 10,
        "robots": 10,
        "vit-market": 1
      }
    },
    {
      "crawler": "ClaudeBot",
      "operator": "Anthropic",
      "purpose": "model-development",
      "declaredRequests": 2906,
      "verifiedRequests": 0,
      "unverifiedRequests": 2906,
      "successfulResponses": 2383,
      "uniquePaths": 1982,
      "firstSeen": "2026-07-16T14:10:50.000Z",
      "lastSeen": "2026-08-14T05:39:35.000Z",
      "pageTypes": {
        "asset": 272,
        "other-page": 1835,
        "robots": 81,
        "home": 3,
        "sitemap": 107,
        "journal-story-or-hub": 404,
        "services": 22,
        "store": 35,
        "vit-market": 122,
        "research": 25
      }
    },
    {
      "crawler": "PerplexityBot",
      "operator": "Perplexity",
      "purpose": "ai-search",
      "declaredRequests": 337,
      "verifiedRequests": 0,
      "unverifiedRequests": 337,
      "successfulResponses": 221,
      "uniquePaths": 269,
      "firstSeen": "2026-07-16T14:10:51.000Z",
      "lastSeen": "2026-08-14T14:50:09.000Z",
      "pageTypes": {
        "asset": 106,
        "other-page": 108,
        "store": 3,
        "journal-story-or-hub": 44,
        "services": 15,
        "home": 6,
        "robots": 16,
        "vit-market": 39
      }
    },
    {
      "crawler": "CCBot",
      "operator": "Common Crawl",
      "purpose": "model-development",
      "declaredRequests": 102,
      "verifiedRequests": 0,
      "unverifiedRequests": 102,
      "successfulResponses": 57,
      "uniquePaths": 95,
      "firstSeen": "2026-07-16T14:10:53.000Z",
      "lastSeen": "2026-08-12T03:36:03.000Z",
      "pageTypes": {
        "asset": 37,
        "other-page": 44,
        "store": 5,
        "journal-story-or-hub": 15,
        "vit-market": 1
      }
    },
    {
      "crawler": "Perplexity-User",
      "operator": "Perplexity",
      "purpose": "user-directed",
      "declaredRequests": 17,
      "verifiedRequests": 0,
      "unverifiedRequests": 17,
      "successfulResponses": 0,
      "uniquePaths": 17,
      "firstSeen": "2026-07-19T02:47:48.000Z",
      "lastSeen": "2026-08-02T22:30:50.000Z",
      "pageTypes": {
        "other-page": 4,
        "asset": 13
      }
    },
    {
      "crawler": "Claude-User",
      "operator": "Anthropic",
      "purpose": "user-directed",
      "declaredRequests": 11,
      "verifiedRequests": 0,
      "unverifiedRequests": 11,
      "successfulResponses": 2,
      "uniquePaths": 11,
      "firstSeen": "2026-07-19T02:47:48.000Z",
      "lastSeen": "2026-07-22T15:36:53.000Z",
      "pageTypes": {
        "other-page": 4,
        "asset": 5,
        "robots": 1,
        "journal-story-or-hub": 1
      }
    },
    {
      "crawler": "Claude-SearchBot",
      "operator": "Anthropic",
      "purpose": "ai-search",
      "declaredRequests": 7,
      "verifiedRequests": 0,
      "unverifiedRequests": 7,
      "successfulResponses": 0,
      "uniquePaths": 7,
      "firstSeen": "2026-07-19T02:47:49.000Z",
      "lastSeen": "2026-07-19T02:47:52.000Z",
      "pageTypes": {
        "asset": 6,
        "other-page": 1
      }
    }
  ],
  "verification": {
    "enabled": true,
    "rule": "OpenAI and Google use official IP-range files retrieved at analysis time; Bing uses forward-confirmed reverse DNS. Unsupported identities remain unverified.",
    "sources": [
      {
        "crawler": "OAI-SearchBot",
        "url": "https://openai.com/searchbot.json",
        "createdAt": "2026-01-02T11:00:00.000000",
        "sha256": "e757b3ed5b84ac71976c02f17d9a77385e7acaf5b50e17c7b60f7e5a9876f102"
      },
      {
        "crawler": "OAI-AdsBot",
        "url": "https://openai.com/adsbot.json",
        "createdAt": "2026-05-12T03:00:00.000000",
        "sha256": "460b800ffb026c5ec1b9540d285f2ade8e1c4a654e65dae315e2bae650b5d74d"
      },
      {
        "crawler": "GPTBot",
        "url": "https://openai.com/gptbot.json",
        "createdAt": "2025-10-30T11:00:00.000000",
        "sha256": "be778b84e1b5436be6ab9d6139ba2b3d6a98c1f581e038ec386e3a7d44cbfe50"
      },
      {
        "crawler": "ChatGPT-User",
        "url": "https://openai.com/chatgpt-user.json",
        "createdAt": "2026-08-14T20:03:38.055838",
        "sha256": "812f11df46d6fcac5779082dba32f7e2d571673f72b3e63b10ef706a25420996"
      },
      {
        "crawler": "Googlebot",
        "url": "https://developers.google.com/static/search/apis/ipranges/googlebot.json",
        "createdAt": "2026-08-14T14:45:49.000000",
        "sha256": "b0831053587186e3c282cecce9ee606d2a0d43d045ba9761dea61615c97a0425"
      },
      {
        "crawler": "bingbot",
        "url": "https://blogs.bing.com/webmaster/August-2012/How-to-Verify-that-Bingbot-is-Bingbot",
        "method": "forward-confirmed reverse DNS"
      }
    ]
  },
  "privacy": "No source IP, query value, raw user agent, request ID or individual visitor record is included in this public aggregate.",
  "publicationGate": "The fixed 30-day evidence window is frozen and automated integrity checks passed. Independent editorial review remains required before this study becomes indexable.",
  "finalization": {
    "frozenAt": "2026-08-14T21:31:22.000Z",
    "integrityChecks": "passed",
    "editorialReview": "pending"
  }
}