{"slug":"check-robots-txt-crawl-rules-a8bc2b50","name":"Check robots.txt crawl rules","summary":"Fetch a site's robots.txt and evaluate crawl permissions: given a URL (plus optional extra paths) and a user-agent, return whether each path is allowed or disallowed, the rule that matched, the user-agent group, the crawl-delay and any declared sitemaps. Implements the Robots Exclusion Protocol (RFC 9309) with longest-match-wins, Allow-over-Disallow tie-breaking and * / $ wildcards.","status":"live","indexable":true,"listed_since":"2026-10-25T17:52:39.208110Z","sources":[{"source":"x402_bazaar","external_id":"https://web.openverbs.com/v1/robots","owner_address":"0xb3f4de103f34b99f0da19b4f16d6b53a14c2194a","seen_at":"2026-10-27T21:41:48.753848Z","removed_at":null}],"endpoints":[{"url":"https://web.openverbs.com/v1/robots","host":"web.openverbs.com","protocol":"x402","state":"live","last_probe_at":"2026-10-28T01:35:52.462302Z","last_ok_at":"2026-10-28T01:35:52.462302Z","tools_count":null,"tools_changed_at":null,"uptime_24h":1.0,"uptime_30d":1.0}],"also_listed_as":[{"slug":"web-openverbs-com-1f1d89f7","name":"web.openverbs.com"}],"payment_options":[{"endpoint_url":"https://web.openverbs.com/v1/robots","network":"eip155:8453","scheme":"exact","rail":"exact","asset":"0x833589fcd6edb6e08f4c7c32d4f71b54bda02913","coin":"USDC","asset_label":"USD Coin","amount":"6000","pay_to":"0xb3f4de103f34b99f0da19b4f16d6b53a14c2194a","observed_at":"2026-10-28T01:35:52.462302Z"}],"usage":{"source":"x402_bazaar","unique_payers_30d":6,"paid_calls_30d":7,"last_paid_at":"2026-10-25T21:58:55.500Z","reported_at":"2026-10-27T21:41:48.753848Z"},"assessment":{"method":"2026.10.2","checks":[{"id":"answers","state":"pass","reason":"Answered a check 9 hours ago."},{"id":"owner","state":"fail","reason":"No one has proven control of its domain or of the wallet it is paid at."},{"id":"track_record","state":"fail","reason":"Listed for 1 days; 30 are needed."},{"id":"real_customers","state":"fail","reason":"6 different wallets paid it in 30 days, as the x402 Bazaar reports (not yet counted by addrbook); 10 are needed."},{"id":"clean_scan","state":"not_applicable","reason":"Not an MCP server."}],"passed":1,"applicable":4,"contradicted":false,"computed_at":"2026-10-27T06:02:48.883519Z"},"identity":{"endpoint_domains":["openverbs.com"],"proven_domains":[],"proven_wallets":[],"namesakes":[],"namesake_count":0,"identity_checks":[],"wallet_attestations":[],"human_backings":[]},"replies":[],"customers":null,"scan":null,"notice":"Not affiliated with addrbook — claim or request removal."}