{"slug":"robots-txt-sitemap-analyzer-e3c8e18a","name":"robots.txt & Sitemap Analyzer","summary":"Give it domains. For each it parses robots.txt (user-agents, allow/disallow, crawl-delay, declared sitemaps, blocks-all) and the XML sitemap(s) (URL counts, nested sitemaps, lastmod coverage), and flags crawlability issues - with a signal and findings. Keyless, read-only. Up to 3 items per call; failed lookups are not charged.","status":"live","headings":[{"slug":"web-scraping","title":"Web scraping & browsers"}],"indexable":true,"listed_since":"2026-10-25T19:51:00.005415Z","sources":[{"source":"x402_bazaar","external_id":"https://agent-data.centorprat.workers.dev/tools/robots-sitemap-analyzer","owner_address":"0x8d38d6026cf3a9805de9c5f0611cee95b532b120","seen_at":"2026-11-10T10:13:40.634563Z","removed_at":null}],"endpoints":[{"url":"https://agent-data.centorprat.workers.dev/tools/robots-sitemap-analyzer","host":"agent-data.centorprat.workers.dev","protocol":"x402","state":"live","last_probe_at":"2026-11-10T23:58:47.362447Z","last_ok_at":"2026-11-10T23:58:47.362447Z","tools_count":null,"tools_changed_at":null,"uptime_24h":1.0,"uptime_30d":1.0}],"also_listed_as":[{"slug":"agent-data-centorprat-workers-dev-f469b4b3","name":"agent-data.centorprat.workers.dev"}],"payment_options":[{"endpoint_url":"https://agent-data.centorprat.workers.dev/tools/robots-sitemap-analyzer","network":"eip155:8453","scheme":"exact","rail":"exact","asset":"0x833589fcd6edb6e08f4c7c32d4f71b54bda02913","coin":"USDC","asset_label":"USD Coin","amount":"20000","pay_to":"0x8d38d6026cf3a9805de9c5f0611cee95b532b120","observed_at":"2026-11-10T23:58:47.362447Z"},{"endpoint_url":"https://agent-data.centorprat.workers.dev/tools/robots-sitemap-analyzer","network":"eip155:137","scheme":"exact","rail":"exact","asset":"0x3c499c542cef5e3811e1192ce70d8cc03d5c3359","coin":"USDC","asset_label":"USD Coin","amount":"20000","pay_to":"0x8d38d6026cf3a9805de9c5f0611cee95b532b120","observed_at":"2026-11-10T23:58:47.362447Z"},{"endpoint_url":"https://agent-data.centorprat.workers.dev/tools/robots-sitemap-analyzer","network":"solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp","scheme":"exact","rail":"exact","asset":"EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v","coin":"USDC","asset_label":null,"amount":"20000","pay_to":"3644DrcePengCVjznEa59Dm7Mu7Acr522NvYyapHRHQh","observed_at":"2026-11-10T23:58:47.362447Z"}],"usage":{"source":"x402_bazaar","unique_payers_30d":2,"paid_calls_30d":4,"last_paid_at":"2026-10-25T17:17:41.320Z","reported_at":"2026-11-10T10:13:40.634563Z"},"assessment":{"method":"2026.11.3","checks":[{"id":"answers","state":"pass","reason":"Answered a check within the last hour."},{"id":"owner","state":"fail","reason":"No one has proven control of its domain or of the wallet it is paid at."},{"id":"track_record","state":"fail","reason":"Listed for 16 days; 30 are needed."},{"id":"real_customers","state":"fail","reason":"2 different wallets paid it in 30 days, as the x402 Bazaar reports (not yet counted by addrbook); 10 are needed."},{"id":"clean_scan","state":"not_applicable","reason":"Not an MCP server."}],"passed":1,"applicable":4,"contradicted":false,"computed_at":"2026-11-11T00:32:54.009113Z"},"identity":{"endpoint_domains":["centorprat.workers.dev"],"proven_domains":[],"proven_wallets":[],"namesakes":[],"namesake_count":0,"identity_checks":[],"wallet_attestations":[],"human_backings":[]},"replies":[],"customers":null,"scan":null,"notice":"Not affiliated with addrbook — claim or request removal."}