{"id":278569,"recorded_at":"2026-10-26T19:14:37.672431Z","observed_at":"2026-10-26T19:14:37.672375Z","subject_kind":"source_record","subject":"x402_bazaar:https://shop.lordevo.com/api/quick/can-fetch","kind":"listing","method":"sync.x402_bazaar/1","body":{"endpoints":[["x402","https://shop.lordevo.com/api/quick/can-fetch"]],"name":"Lordevo","owner_address":"0x0876a79bb5a88bf338e6b47b4530359527b01d73","raw":{"accepts":[{"amount":"10000","asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","extra":{"name":"USD Coin","version":"2"},"maxTimeoutSeconds":120,"network":"eip155:8453","payTo":"0x0876A79Bb5A88Bf338E6b47b4530359527b01d73","scheme":"exact"}],"description":"Can I fetch this URL? robots.txt + AI-crawler permission gate: Pre-fetch gate for agents and scrapers: is this exact URL allowed by robots.txt (RFC 9309: missing = allow, 5xx/unreachable = disallow) for generic agents and 16 named AI agents/crawlers (ChatGPT-User, Claude-User, PerplexityBot, GPTBot,","extensions":{"bazaar":{"info":{"input":{"method":"GET","queryParams":{"url":"https://example.com"},"type":"http"},"output":{"example":{"agents":{"*":true,"CCBot":false,"ChatGPT-User":true,"Claude-User":true,"ClaudeBot":true,"GPTBot":false,"PerplexityBot":true},"allowed":true,"content_signal":{"ai-input":false,"ai-train":false,"search":true},"crawl_delay_seconds":null,"noai":true,"origin":"https://example.com","page":{"content_type":"text/html; charset=utf-8","final_url":"https://example.com/blog/post","http_status":200,"x_robots_tag":[]},"path":"/blog/post","robots_http_status":200,"robots_txt":"present","robots_url":"https://example.com/robots.txt","sitemaps":["https://example.com/sitemap.xml"],"summary":"Allowed: example.com/blog/post for generic crawlers/agents. Blocked for: GPTBot, CCBot. The site signals no AI use (noai / Content-Signal).","tool":"can-fetch","url":"https://example.com/blog/post","verdict":"allowed_but_no_ai_use"},"type":"json"}},"schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"input":{"additionalProperties":false,"properties":{"method":{"enum":["GET"],"type":"string"},"queryParams":{"properties":{"url":{"description":"public http(s) URL or bare domain","type":"string"}},"required":["url"],"type":"object"},"type":{"const":"http","type":"string"}},"required":["type","method"],"type":"object"},"output":{"properties":{"example":{"properties":{"agents":{"additionalProperties":{"type":"boolean"},"type":"object"},"allowed":{"type":"boolean"},"content_signal":{"type":["object","null"]},"crawl_delay_seconds":{"type":["number","null"]},"noai":{"type":"boolean"},"origin":{"type":"string"},"page":{"type":["object","null"]},"path":{"type":"string"},"robots_http_status":{"type":["integer","null"]},"robots_txt":{"enum":["present","missing","unreachable"],"type":"string"},"sitemaps":{"items":{"type":"string"},"type":"array"},"summary":{"type":"string"},"tool":{"type":"string"},"url":{"type":"string"},"verdict":{"enum":["allowed","disallowed","allowed_but_no_ai_use","allowed_with_crawl_delay"],"type":"string"}},"required":["tool","url","verdict","allowed","agents","robots_txt"],"type":"object"},"type":{"type":"string"}},"required":["type"],"type":"object"}},"required":["input"],"type":"object"}}},"iconUrl":"https://res.cloudinary.com/bdb-prod/image/upload/c_fill,ar_1:1,w_256,h_256,f_auto,q_auto/v1793001371/bazaar-icons/2f77a59e7f2d5bff.png","lastUpdated":"2026-10-26T17:40:08.756Z","quality":{"l30DaysTotalCalls":1,"l30DaysUniquePayers":1,"lastCalledAt":"2026-10-26T17:40:07.717Z"},"resource":"https://shop.lordevo.com/api/quick/can-fetch","serviceName":"Lordevo","tags":["robots.txt","can i crawl","crawl permission","ai crawlers","x-robots-tag"],"type":"http","x402Version":2},"source":"x402_bazaar","summary":"Can I fetch this URL? robots.txt + AI-crawler permission gate: Pre-fetch gate for agents and scrapers: is this exact URL allowed by robots.txt (RFC 9309: missing = allow, 5xx/unreachable = disallow) for generic agents and 16 named AI agents/crawlers (ChatGPT-User, Claude-User, PerplexityBot, GPTBot,"},"body_sha256":"c2104f7fc743fb5d3b2851491044f4eb738d0bfb9bbc26f4b30863a4639b1b7e","prev_id":272942,"leaf":"0x30acb8043de7ce4709343d924356cbf244ac54b3f5b01383a7ae9af424eb833a","leaf_text":"{\"body\":\"c2104f7fc743fb5d3b2851491044f4eb738d0bfb9bbc26f4b30863a4639b1b7e\",\"id\":278569,\"kind\":\"listing\",\"method\":\"sync.x402_bazaar/1\",\"observed_us\":1793042077672375,\"prev\":272942,\"subject\":\"x402_bazaar:https://shop.lordevo.com/api/quick/can-fetch\",\"subject_kind\":\"source_record\"}","day":{"day":"2026-10-26","leaves":296695,"root":"0xf43e8dcc3d451a4788b26be904c55e8c4c0fa75c9666796c51db38d37e2c0df8","chained":"0x702d5ac508f08b0d1accf36af25822d8e5660367494edd17a642fb61e158a57a","anchor_chain":"eip155:5042002","anchor_tx":"0x495162f8936e19f0039918feebe6c307f327cd6bf8c37219620b1970041fa9ae","anchor_block":69251483},"proof":["0x30ace064d2557d7521e16278e0eee377de806c4d1f70e2c3cca1b27c3472ea2a","0x3ec74ba99afe743f2cc9f74677e45d58e0ffdcf52f1c63c5c371a7bf8a82b52f","0xc13f41e032e94316c36186996a46fa2bfce1bf3f19392b65b61faac71e4d1091","0xdd809999bd782f4ab9ec975f546fa0203d218671e888a37a3c763f16abb0fd1b","0x650ad1ef203c936bf4de52b2db5afd80b60839a8df3ee2bc11847b09634f30aa","0x6f7e8baf49d53a3efcbb34cffa94318b7fadf365bc2e4ef286693e18a515ce5d","0xf44c0cebdcc3b4f1556684339c8beea4e4b0bd926be3680dc2a486d42b6e44ae","0x6752cb22f2e17d332b0bcad5f02d609e44802bbea0355fc500efd0939b0a61f2","0x25ba1b9145475ea9ef33090e367d91a0fd38d074fef51cf5ad78e396de8c39ee","0x0886695ce8b00ff9cb3b286883a23d258a3fbdfe296f9e24a8b5f848bccd41cb","0x66b4d790d6ae725a0af009d5a6edacaec5a6147d597bc8410a97be11d56a00d1","0xd85c7204d76d26255c94ef4caa12c05addd08238caa5cdc3888ea60b74e6b940","0x36a44931b230976c04691d90bf3a25ed4111d74c78b672850d8025a911ca88dd","0x683955a5b2250ad50568d022a6580d8739b6aff1356779d72a773b088e7c289c","0x610738e0c43687aded83c613c73de8b5226c73c375858a167436461c522ea213","0x06d71bfa7d8b41d62c8836efc2c601e1cc3772d9453596b2ae4950887f3310ca","0x8c756542e482157df0c36464d75c937b845e4423dbbcfad08f0cf8d5a16b9a13","0x20704208aa6923d6b2450ef74ede87a6e0265d46447926038fe9506e9e154b2d","0x9b9ce5b74b9bab5121612ef6b64920a8d453cbf311096a423eef9702747e868a"]}