{"schema":"ratradar/v1","purpose":"Rat Radar tells a person or an AI agent whether a website will let them collect its data, and the best way to do it.","generated":"2026-10-11","source":"https://ratlabs.tech/radar/","license":"Free to use. Please credit Rat Radar by RatLabs.","sites_probed":653,"snapshots":["2026-10-08","2026-10-10","2026-10-11"],"verdicts":{"open":346,"hostile":128,"blocked":129,"guarded":50},"headline":{"percent_blocking_any_ai_crawler":26,"percent_blocking_three_or_more_ai_crawlers":20,"percent_disallowing_all_crawlers":5,"percent_with_sitemap":54,"percent_with_llms_txt":17,"percent_with_json_ld":32,"percent_refusing_headless_browsers_but_not_plain_http":13},"ai_crawler_blocks":{"GPTBot":117,"ChatGPT-User":66,"OAI-SearchBot":46,"ClaudeBot":110,"Claude-Web":67,"CCBot":120,"PerplexityBot":83,"Google-Extended":97,"Bytespider":111,"cohere-ai":76,"Meta-ExternalAgent":92,"anthropic-ai":82,"Applebot-Extended":81,"Amazonbot":76},"ai_crawler_policy":{"block_gptbot_allow_oai_searchbot":72,"block_claudebot_not_gptbot":18,"block_gptbot_not_claudebot":25,"block_only_retired_anthropic_ai":["abcnews.go.com","elmundo.es","repubblica.it","trustpilot.com"],"note":"GPTBot collects training data; OAI-SearchBot fetches for ChatGPT search. anthropic-ai is a retired name: blocking it without ClaudeBot does not block Claude's crawler."},"eu_ai_training":{"none":359,"some":252,"strong":42,"percent_with_any_signal":45,"by_signal":{"robots_txt":192,"captcha":142,"login_wall":2},"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"Signals, not a legal finding. They matter when the data includes personal data and is used to train generative AI."},"bot_protection_vendors":{"AWS WAF":24,"CloudFront":162,"Cloudflare":249,"PerimeterX":23,"Fastly":83,"DataDome":35,"Akamai":108,"Kasada":4,"Imperva":2},"cdn":{"Google":4,"CloudFront":127,"Cloudflare":182,"GitHub":1,"Fastly":65,"Akamai":71,"Vercel":11,"Netlify":4},"by_category":{"search":{"sites":15,"open":9,"guarded":0,"hostile":6,"blocked":0,"avg_hostility":25},"shop":{"sites":105,"open":42,"guarded":11,"hostile":20,"blocked":32,"avg_hostility":46},"social":{"sites":40,"open":17,"guarded":1,"hostile":18,"blocked":4,"avg_hostility":38},"dev":{"sites":52,"open":48,"guarded":1,"hostile":1,"blocked":2,"avg_hostility":9},"news":{"sites":80,"open":49,"guarded":9,"hostile":11,"blocked":11,"avg_hostility":28},"property":{"sites":29,"open":5,"guarded":0,"hostile":10,"blocked":14,"avg_hostility":65},"market":{"sites":26,"open":10,"guarded":2,"hostile":4,"blocked":10,"avg_hostility":49},"jobs":{"sites":33,"open":15,"guarded":2,"hostile":4,"blocked":12,"avg_hostility":47},"reviews":{"sites":20,"open":8,"guarded":0,"hostile":2,"blocked":10,"avg_hostility":57},"travel":{"sites":44,"open":16,"guarded":12,"hostile":8,"blocked":8,"avg_hostility":40},"ai":{"sites":21,"open":15,"guarded":0,"hostile":3,"blocked":3,"avg_hostility":27},"gov":{"sites":59,"open":41,"guarded":2,"hostile":10,"blocked":6,"avg_hostility":24},"food":{"sites":24,"open":9,"guarded":5,"hostile":6,"blocked":4,"avg_hostility":40},"saas":{"sites":15,"open":13,"guarded":0,"hostile":2,"blocked":0,"avg_hostility":13},"media":{"sites":23,"open":14,"guarded":2,"hostile":6,"blocked":1,"avg_hostility":28},"edu":{"sites":23,"open":13,"guarded":0,"hostile":6,"blocked":4,"avg_hostility":35},"finance":{"sites":34,"open":21,"guarded":1,"hostile":8,"blocked":4,"avg_hostility":30},"auto":{"sites":10,"open":1,"guarded":2,"hostile":3,"blocked":4,"avg_hostility":62}},"by_country":{"United States":{"sites":336,"avg_hostility":35},"Denmark":{"sites":2,"avg_hostility":49},"Netherlands":{"sites":13,"avg_hostility":24},"Canada":{"sites":14,"avg_hostility":29},"United Arab Emirates":{"sites":11,"avg_hostility":52},"United Kingdom":{"sites":51,"avg_hostility":41},"Germany":{"sites":34,"avg_hostility":44},"France":{"sites":17,"avg_hostility":37},"Russia":{"sites":7,"avg_hostility":20},"China":{"sites":22,"avg_hostility":21},"Japan":{"sites":12,"avg_hostility":22},"South Korea":{"sites":9,"avg_hostility":46},"India":{"sites":14,"avg_hostility":45},"Argentina":{"sites":2,"avg_hostility":31},"Brazil":{"sites":5,"avg_hostility":37},"South Africa":{"sites":4,"avg_hostility":40},"Nigeria":{"sites":2,"avg_hostility":54},"Australia":{"sites":12,"avg_hostility":37},"Spain":{"sites":10,"avg_hostility":54},"Sweden":{"sites":6,"avg_hostility":28},"Poland":{"sites":3,"avg_hostility":34},"Ireland":{"sites":6,"avg_hostility":22},"Singapore":{"sites":13,"avg_hostility":41},"Indonesia":{"sites":3,"avg_hostility":27},"Colombia":{"sites":1,"avg_hostility":2},"Israel":{"sites":3,"avg_hostility":39},"Malaysia":{"sites":1,"avg_hostility":100},"Italy":{"sites":5,"avg_hostility":34},"Hong Kong":{"sites":1,"avg_hostility":6},"Switzerland":{"sites":3,"avg_hostility":6},"Belgium":{"sites":3,"avg_hostility":6},"New Zealand":{"sites":3,"avg_hostility":54},"Norway":{"sites":2,"avg_hostility":5},"Chile":{"sites":1,"avg_hostility":6},"Mexico":{"sites":2,"avg_hostility":65},"Pakistan":{"sites":2,"avg_hostility":30},"Turkey":{"sites":3,"avg_hostility":41},"Ukraine":{"sites":3,"avg_hostility":69},"Romania":{"sites":1,"avg_hostility":74},"Greece":{"sites":1,"avg_hostility":100},"Czechia":{"sites":3,"avg_hostility":29},"Kenya":{"sites":2,"avg_hostility":53},"Finland":{"sites":2,"avg_hostility":10},"Portugal":{"sites":2,"avg_hostility":30},"Philippines":{"sites":1,"avg_hostility":55}},"easiest":["central.sonatype.com","cloud.google.com","codeberg.org","cohere.com","duckduckgo.com","find-and-update.company-information.service.gov.uk","go.dev","habr.com","hashnode.com","hatena.ne.jp"],"hardest":["www2.hm.com","www.zoopla.co.uk","www.zara.com","www.yelp.com","www.yellowpages.com","www.wsj.com","www.viator.com","www.very.co.uk","www.trustradius.com","www.trivago.com"]}