{"slug":"api-x402node-dev-web-robots-296a1220","name":"api.x402node.dev/web/robots","summary":"Fetch and parse robots.txt for a domain. Returns parsed allow/disallow rules per user-agent, sitemap URLs, crawl-delay directives, and raw content. Built for AI agents planning ethical web crawling and SEO audit. Use ?domain=example.com. Accepts payment on Base or Solana — either network works.","status":"live","headings":[{"slug":"web-scraping","title":"Web scraping & browsers"}],"indexable":true,"listed_since":"2026-10-25T17:57:32.728600Z","sources":[{"source":"x402_bazaar","external_id":"https://api.x402node.dev/web/robots","owner_address":"0x4466d4a84b7c49a6a094ec6eef4a0712d6dd125e","seen_at":"2026-11-10T07:40:35.003619Z","removed_at":null}],"endpoints":[{"url":"https://api.x402node.dev/web/robots","host":"api.x402node.dev","protocol":"x402","state":"live","last_probe_at":"2026-11-09T05:30:40.747869Z","last_ok_at":"2026-11-09T05:30:40.747869Z","tools_count":null,"tools_changed_at":null,"uptime_24h":null,"uptime_30d":1.0}],"also_listed_as":[{"slug":"api-x402node-dev-e7249e95","name":"api.x402node.dev"}],"payment_options":[{"endpoint_url":"https://api.x402node.dev/web/robots","network":"eip155:8453","scheme":"exact","rail":"exact","asset":"0x833589fcd6edb6e08f4c7c32d4f71b54bda02913","coin":"USDC","asset_label":"USD Coin","amount":"5000","pay_to":"0x4466d4a84b7c49a6a094ec6eef4a0712d6dd125e","observed_at":"2026-11-09T05:30:40.747869Z"},{"endpoint_url":"https://api.x402node.dev/web/robots","network":"solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp","scheme":"exact","rail":"exact","asset":"EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v","coin":"USDC","asset_label":null,"amount":"5000","pay_to":"9urRvUx69HqAopg1JGV9P9wR7JdKoztzdkuNyJ8Eo1up","observed_at":"2026-11-09T05:30:40.747869Z"}],"usage":{"source":"x402_bazaar","unique_payers_30d":1,"paid_calls_30d":1,"last_paid_at":"2026-10-24T22:47:21.943Z","reported_at":"2026-11-10T07:40:35.003619Z"},"assessment":{"method":"2026.10.2","checks":[{"id":"answers","state":"pass","reason":"Answered a check 19 hours ago."},{"id":"owner","state":"fail","reason":"No one has proven control of its domain or of the wallet it is paid at."},{"id":"track_record","state":"fail","reason":"Listed for 15 days; 30 are needed."},{"id":"real_customers","state":"fail","reason":"1 different wallets paid it in 30 days, as the x402 Bazaar reports (not yet counted by addrbook); 10 are needed."},{"id":"clean_scan","state":"not_applicable","reason":"Not an MCP server."}],"passed":1,"applicable":4,"contradicted":false,"computed_at":"2026-11-10T00:34:28.714017Z"},"identity":{"endpoint_domains":["x402node.dev"],"proven_domains":[],"proven_wallets":[],"namesakes":[],"namesake_count":0,"identity_checks":[],"wallet_attestations":[],"human_backings":[]},"replies":[],"customers":null,"scan":null,"notice":"Not affiliated with addrbook — claim or request removal."}