{"slug":"api-strale-io-x402-robots-txt-parse-7a311087","name":"api.strale.io/x402/robots-txt-parse","summary":"Robots.txt Parser: Fetch and read a site's robots.txt: rules for each crawler, sitemaps, crawl delays, and whether all crawlers are blocked.","status":"live","headings":[{"slug":"web-scraping","title":"Web scraping & browsers"}],"indexable":true,"listed_since":"2026-10-25T17:53:33.961953Z","sources":[{"source":"x402_bazaar","external_id":"https://api.strale.io/x402/robots-txt-parse","owner_address":null,"seen_at":"2026-11-18T10:41:15.711797Z","removed_at":null}],"endpoints":[{"url":"https://api.strale.io/x402/robots-txt-parse","host":"api.strale.io","protocol":"x402","state":"live","last_probe_at":"2026-11-18T13:36:43.027456Z","last_ok_at":"2026-11-18T13:36:43.027456Z","tools_count":null,"tools_changed_at":null,"uptime_24h":1.0,"uptime_30d":1.0}],"also_listed_as":[{"slug":"api-strale-io-07f24610","name":"api.strale.io"}],"payment_options":[{"endpoint_url":"https://api.strale.io/x402/robots-txt-parse","network":"eip155:8453","scheme":"exact","rail":"exact","asset":"0x833589fcd6edb6e08f4c7c32d4f71b54bda02913","coin":"USDC","asset_label":"USD Coin","amount":"21600","pay_to":"0x66d7c2f952362bfb24fd7f02a9bec9c754ea83bc","observed_at":"2026-11-18T13:36:43.027456Z"}],"usage":{"source":"x402_bazaar","unique_payers_30d":1,"paid_calls_30d":21,"last_paid_at":"2026-11-18T06:17:09.053Z","reported_at":"2026-11-18T10:41:15.711797Z"},"assessment":{"method":"2026.11.4","checks":[{"id":"answers","state":"pass","reason":"Answered a check 3 hours ago."},{"id":"owner","state":"fail","reason":"No one has proven control of its domain or of the wallet it is paid at."},{"id":"track_record","state":"fail","reason":"Listed for 23 days; 30 are needed."},{"id":"real_customers","state":"fail","reason":"1 different wallets paid it in 30 days, as the x402 Bazaar reports (not yet counted by addrbook); 10 are needed."},{"id":"clean_scan","state":"not_applicable","reason":"Not an MCP server."}],"passed":1,"applicable":4,"contradicted":false,"computed_at":"2026-11-18T10:41:42.149800Z"},"identity":{"endpoint_domains":["strale.io"],"proven_domains":[],"proven_wallets":[],"namesakes":[],"namesake_count":0,"identity_checks":[],"wallet_attestations":[],"human_backings":[]},"replies":[],"customers":null,"scan":null,"notice":"Not affiliated with addrbook — claim or request removal."}