{"id":125541,"recorded_at":"2026-10-26T14:40:26.452663Z","observed_at":"2026-10-26T14:40:26.452628Z","subject_kind":"source_record","subject":"x402_bazaar:https://x402.agentharbor.agency/ai-crawlers","kind":"listing","method":"baseline/1","body":{"endpoints":[["x402","https://x402.agentharbor.agency/ai-crawlers"]],"name":"ah-ai-crawlers","owner_address":"0x840c829530df7b4ccf700fa3a0060cf2c80b014e","raw":{"accepts":[{"amount":"10000","asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","extra":{"name":"USD Coin","version":"2"},"maxTimeoutSeconds":300,"network":"eip155:8453","payTo":"0x840C829530DF7B4CCF700Fa3A0060cF2C80b014e","scheme":"exact"}],"description":"Given a website domain, reports which AI crawlers may fetch its home page under its robots.txt (RFC 9309): GPTBot, OAI-SearchBot, ChatGPT-User, ClaudeBot, Claude-SearchBot, PerplexityBot, Google-Extended, Applebot-Extended, CCBot and more, each allowed or blocked, plus Googlebot, listed sitemaps, whether /llms.txt is published and page-level noindex/nosnippet, with fixes. Static check, no browser. Not charged if robots.txt cannot be read (no response, or a bot block).","extensions":{"bazaar":{"info":{"input":{"method":"GET","queryParams":{"domain":"agentharbor.agency"},"type":"http"},"output":{"example":{"data":{"ai_crawlers":{"bot_access":{"Amazonbot":"allowed","Applebot-Extended":"allowed","Bytespider":"allowed","CCBot":"allowed","ChatGPT-User":"allowed","Claude-SearchBot":"allowed","Claude-User":"allowed","ClaudeBot":"allowed","GPTBot":"allowed","Google-Extended":"allowed","Meta-ExternalAgent":"allowed","OAI-SearchBot":"allowed","PerplexityBot":"allowed"},"googlebot":"allowed","llms_txt":"present","robots_txt":"present","sitemaps":[]},"findings":[{"confidence":"confirmed","evidence":[],"fix":"Add 'Sitemap: https://your-domain/sitemap.xml'.","id":"ai.robots.no_sitemap","severity":"low","title":"robots.txt does not list a sitemap"},{"confidence":"confirmed","evidence":["# Agent Harbor"],"fix":"","id":"ai.llms_txt.present","severity":"info","title":"llms.txt is present"}],"notes":[]},"domain":"agentharbor.agency","price_paid_usdc":0.01,"provider":"Agent Harbor — deterministic audit engine","tier":"ai-crawlers"},"type":"json"}},"schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"input":{"additionalProperties":false,"properties":{"method":{"enum":["GET","HEAD","DELETE"],"type":"string"},"queryParams":{"additionalProperties":false,"properties":{"domain":{"description":"Bare public hostname, e.g. example.com","maxLength":253,"pattern":"^[A-Za-z0-9]([A-Za-z0-9-]{0,61}[A-Za-z0-9])?(\\.[A-Za-z0-9]([A-Za-z0-9-]{0,61}[A-Za-z0-9])?)+$","type":"string"}},"required":["domain"],"type":"object"},"type":{"const":"http","type":"string"}},"required":["type","method"],"type":"object"},"output":{"properties":{"example":{"type":"object"},"type":{"type":"string"}},"required":["type"],"type":"object"}},"required":["input"],"type":"object"}}},"lastUpdated":"2026-10-25T16:50:47.06Z","quality":{"l30DaysTotalCalls":4,"l30DaysUniquePayers":4,"lastCalledAt":"2026-10-25T16:50:46.605Z"},"resource":"https://x402.agentharbor.agency/ai-crawlers","serviceName":"ah-ai-crawlers","tags":["ai-crawlers","robots-txt","llms-txt","gptbot","seo"],"type":"http","x402Version":2},"source":"x402_bazaar","summary":"Given a website domain, reports which AI crawlers may fetch its home page under its robots.txt (RFC 9309): GPTBot, OAI-SearchBot, ChatGPT-User, ClaudeBot, Claude-SearchBot, PerplexityBot, Google-Extended, Applebot-Extended, CCBot and more, each allowed or blocked, plus Googlebot, listed sitemaps, whether /llms.txt is published and page-level noindex/nosnippet, with fixes. Static check, no browser. Not charged if robots.txt cannot be read (no response, or a bot block)."},"body_sha256":"d13f6c6eaefdb05d3fd3d5bbe5235d27dfe3fc9e34f5dfa1efa94b8053d5ae82","prev_id":null,"leaf":"0x449d76d3a17ba53ad46bdb7bae67c3ac6c9efe461f1b34bfba818131eeeaa860","leaf_text":"{\"body\":\"d13f6c6eaefdb05d3fd3d5bbe5235d27dfe3fc9e34f5dfa1efa94b8053d5ae82\",\"id\":125541,\"kind\":\"listing\",\"method\":\"baseline/1\",\"observed_us\":1793025626452628,\"prev\":null,\"subject\":\"x402_bazaar:https://x402.agentharbor.agency/ai-crawlers\",\"subject_kind\":\"source_record\"}","day":{"day":"2026-10-26","leaves":296695,"root":"0xf43e8dcc3d451a4788b26be904c55e8c4c0fa75c9666796c51db38d37e2c0df8","chained":"0x702d5ac508f08b0d1accf36af25822d8e5660367494edd17a642fb61e158a57a","anchor_chain":"eip155:5042002","anchor_tx":"0x495162f8936e19f0039918feebe6c307f327cd6bf8c37219620b1970041fa9ae","anchor_block":69251483},"proof":["0x449d91a94b130ae2740702aed7861223d43c18054f2bb0c427105d6dda409a6b","0xedb5f8e827c81d9dcf6ba6e3e428e6f86b85ba3b9e726d5fac19d9de502fa2b4","0x95eb39258efb5a397804f1f32ac098f0ec0d59ee8f58ef5f0a8a648e31e0cb55","0xe6f57172ac4e26f843f9fb769dc02c08f708eadb4ee5d21a654546792cded350","0x3162481376527af03146279a21d040b3f333efe92d85dcf5d6bedd5536d4f44d","0x0153558962c87ae157f88ad17d7043d00ff993ec1571fed5a420dd7b66767fc1","0x4e40d83806fb4a294c876ce3168d585037c6fc7acf6013388a1167af509d48bf","0x3e09a8531f2a4ccef69569b8b45c911059f13de7ab4d85b3e97ed835ca3e2a1e","0x93f30edd2bcc7804a91d13aae0f67c862902db2481f1e19ada629077fd8aa6e9","0xde181bf1fcb403c809d49f37f3f74a5547c660ce89eda5bb438ddeecff34c900","0x1204d0e3e739db3e40f40f102e1314028f064872da22239d712e0ef4cff7deed","0x2b1b47804e83988d7fd5ed8179b2334f3f43b318109b54142f6c6bd749076f9e","0x54b29089c0a3cbbd6ddbe58944696110984dd5f418275cb1b77f957c56da8eab","0x8b27a4c21bf4ba962fd008c8f4300bc6953764a5032b9ab0cfe696b318c45326","0x4b52a97f7a80ff7778a702b85267b8d2e5c93ce160942656435c29ae7be68bf4","0x2e6a32eb32761c210560e852405e2f865f958ddbe01027894f8b9c1b3433b9f0","0x9a83be955a206deb65b995adfd92c58fa31c95fbbf926bd199d300e70b08b26d","0x20704208aa6923d6b2450ef74ede87a6e0265d46447926038fe9506e9e154b2d","0x9b9ce5b74b9bab5121612ef6b64920a8d453cbf311096a423eef9702747e868a"]}