{"id":272942,"recorded_at":"2026-10-26T18:24:17.894015Z","observed_at":"2026-10-26T18:24:17.893956Z","subject_kind":"source_record","subject":"x402_bazaar:https://shop.lordevo.com/api/quick/can-fetch","kind":"listing","method":"sync.x402_bazaar/1","body":{"endpoints":[["x402","https://shop.lordevo.com/api/quick/can-fetch"]],"name":"Lordevo","owner_address":"0x0876a79bb5a88bf338e6b47b4530359527b01d73","raw":{"accepts":[{"amount":"10000","asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","extra":{"name":"USD Coin","version":"2"},"maxTimeoutSeconds":120,"network":"eip155:8453","payTo":"0x0876A79Bb5A88Bf338E6b47b4530359527b01d73","scheme":"exact"}],"description":"Can I fetch this URL? robots.txt + AI-crawler permission gate: Pre-fetch gate for agents and scrapers: is this exact URL allowed by robots.txt (RFC 9309: missing = allow, 5xx/unreachable = disallow) for generic agents and 16 named AI agents/crawlers (ChatGPT-User, Claude-User, PerplexityBot, GPTBot,","extensions":{"bazaar":{"info":{"input":{"method":"GET","queryParams":{"url":"https://example.com"},"type":"http"},"output":{"example":{"agents":{"*":true,"CCBot":false,"ChatGPT-User":true,"Claude-User":true,"ClaudeBot":true,"GPTBot":false,"PerplexityBot":true},"allowed":true,"content_signal":{"ai-input":false,"ai-train":false,"search":true},"crawl_delay_seconds":null,"noai":true,"origin":"https://example.com","page":{"content_type":"text/html; charset=utf-8","final_url":"https://example.com/blog/post","http_status":200,"x_robots_tag":[]},"path":"/blog/post","robots_http_status":200,"robots_txt":"present","robots_url":"https://example.com/robots.txt","sitemaps":["https://example.com/sitemap.xml"],"summary":"Allowed: example.com/blog/post for generic crawlers/agents. Blocked for: GPTBot, CCBot. The site signals no AI use (noai / Content-Signal).","tool":"can-fetch","url":"https://example.com/blog/post","verdict":"allowed_but_no_ai_use"},"type":"json"}},"schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"input":{"additionalProperties":false,"properties":{"method":{"enum":["GET"],"type":"string"},"queryParams":{"properties":{"url":{"description":"public http(s) URL or bare domain","type":"string"}},"required":["url"],"type":"object"},"type":{"const":"http","type":"string"}},"required":["type","method"],"type":"object"},"output":{"properties":{"example":{"properties":{"agents":{"additionalProperties":{"type":"boolean"},"type":"object"},"allowed":{"type":"boolean"},"content_signal":{"type":["object","null"]},"crawl_delay_seconds":{"type":["number","null"]},"noai":{"type":"boolean"},"origin":{"type":"string"},"page":{"type":["object","null"]},"path":{"type":"string"},"robots_http_status":{"type":["integer","null"]},"robots_txt":{"enum":["present","missing","unreachable"],"type":"string"},"sitemaps":{"items":{"type":"string"},"type":"array"},"summary":{"type":"string"},"tool":{"type":"string"},"url":{"type":"string"},"verdict":{"enum":["allowed","disallowed","allowed_but_no_ai_use","allowed_with_crawl_delay"],"type":"string"}},"required":["tool","url","verdict","allowed","agents","robots_txt"],"type":"object"},"type":{"type":"string"}},"required":["type"],"type":"object"}},"required":["input"],"type":"object"}}},"iconUrl":"https://res.cloudinary.com/bdb-prod/image/upload/c_fill,ar_1:1,w_256,h_256,f_auto,q_auto/v1793001371/bazaar-icons/2f77a59e7f2d5bff.png","lastUpdated":"2026-10-26T17:40:08.756Z","quality":{"lastCalledAt":"2026-10-26T17:40:08.756Z"},"resource":"https://shop.lordevo.com/api/quick/can-fetch","serviceName":"Lordevo","tags":["robots.txt","can i crawl","crawl permission","ai crawlers","x-robots-tag"],"type":"http","x402Version":2},"source":"x402_bazaar","summary":"Can I fetch this URL? robots.txt + AI-crawler permission gate: Pre-fetch gate for agents and scrapers: is this exact URL allowed by robots.txt (RFC 9309: missing = allow, 5xx/unreachable = disallow) for generic agents and 16 named AI agents/crawlers (ChatGPT-User, Claude-User, PerplexityBot, GPTBot,"},"body_sha256":"69d85cf9a35a6a7daa8ca8d0c29ae82119ce2ec8c6de12168bb222e1f06d996e","prev_id":null,"leaf":"0x1d4548b0bcbc8acfe993f598b83c164b696181403755586343bb4fe45e57245e","leaf_text":"{\"body\":\"69d85cf9a35a6a7daa8ca8d0c29ae82119ce2ec8c6de12168bb222e1f06d996e\",\"id\":272942,\"kind\":\"listing\",\"method\":\"sync.x402_bazaar/1\",\"observed_us\":1793039057893956,\"prev\":null,\"subject\":\"x402_bazaar:https://shop.lordevo.com/api/quick/can-fetch\",\"subject_kind\":\"source_record\"}","day":{"day":"2026-10-26","leaves":296695,"root":"0xf43e8dcc3d451a4788b26be904c55e8c4c0fa75c9666796c51db38d37e2c0df8","chained":"0x702d5ac508f08b0d1accf36af25822d8e5660367494edd17a642fb61e158a57a","anchor_chain":"eip155:5042002","anchor_tx":"0x495162f8936e19f0039918feebe6c307f327cd6bf8c37219620b1970041fa9ae","anchor_block":69251483},"proof":["0x1d456261acca5caf713dea1c2a3b8106204471d0016d8be21d12682032b1b32f","0xe84db8feb12ae74188adf587448c2bb0f9c21ac424c96da7be9eed9398301f2f","0x16000c32e10fbba10fa9e0c2c9605ef792be9a1be62bb97ed329f3db9306c579","0x797e56f6d3fe674a6309916c2ad440a210833ff3fc956634853530dcd5e23ddf","0x057ab513a85451f022b7e7da5b12b839a89e52f16f1609c26d9a7a85d412638d","0x0dd7419c6ac32980b8bcf8211b818e5b2901cf3043b6e4dfcc8a23a4433cdf10","0xab8873193c8ad250435f1c3818073e2aedc7fd7620306c57205a7763137493c5","0x98b1e9c8fec3eff4dc34e17bd3193c37de0197b70ca005c4b60a8703f75e2631","0x3d86f02bcefd5c503f8a62d4f8a065a11eaac825db5166bd8edd4faaed62c7a3","0x328a19bb2b9e8aa664a31a15df6f2e5a801e6cb13c626e7a570636fb14fb88d3","0x15110e1ab8fe267895da4a2cf012c17f5824a8afca5d2dac1f158d633e16c91c","0x8e3531af6dd328ff9a174dd19751228f67fa0773c68a89edf859dffbd7b098c3","0x42b0d9d5b53af5b0fe99cee10ce29fe484db78b92ce49f97190cc6d098651d47","0x9dfd312f3d4937f7cb144cee3987e8516f222932b4ea286605236d73cfa1ed16","0xc3a6a3ae44ec0da1848b16d40c45d7aac97a67352eea8df18e0f26edabea8a2f","0x06d71bfa7d8b41d62c8836efc2c601e1cc3772d9453596b2ae4950887f3310ca","0x8c756542e482157df0c36464d75c937b845e4423dbbcfad08f0cf8d5a16b9a13","0x20704208aa6923d6b2450ef74ede87a6e0265d46447926038fe9506e9e154b2d","0x9b9ce5b74b9bab5121612ef6b64920a8d453cbf311096a423eef9702747e868a"]}