{"schema":"ratradar/v1","purpose":"Rat Radar tells a person or an AI agent whether a website will let them collect its data, and the best way to do it.","generated":"2026-10-11","source":"https://ratlabs.tech/radar/","license":"Free to use. Please credit Rat Radar by RatLabs.","count":653,"sites":[{"schema":"ratradar.site/v1","host":"www.google.com","name":"Google","category":"search","country":"United States","verdict":"open","hostility_score":20,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls.","Enumerate pages from the sitemap: https://www.google.com/sitemap.xml","A browser load called JSON endpoints that look like data, for example www.google.com/complete/s. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"Google","server":"gws"},"robots":{"found":true,"disallow_all":false,"disallow_rules":169,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.google.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.google.com/complete/s","ogads-pa.clients6.google.com/$rpc/google.internal.onegoogle.asyncdata.v1.AsyncDataService/GetAsyncData"],"inner_page":{"url":"https://www.google.com/intl/de/get/cardboard/apps/","status":404,"json_ld_types":[],"skipped":null,"sitemap":"https://www.google.com/get/sitemap.xml"}},"performance":{"response_ms":5491,"bytes":88954,"redirects":0,"cookies_set":3},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15},{"label":"Slow response (5491 ms)","points":5}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[{"slug":"google-maps-scraper","title":"Google Maps Scraper","url":"https://apify.com/labrat011/google-maps-scraper"},{"slug":"google-news-scraper","title":"Google News Scraper","url":"https://apify.com/labrat011/google-news-scraper"},{"slug":"google-autocomplete-scraper","title":"Google Autocomplete Scraper: Google, YouTube, Amazon, Bing","url":"https://apify.com/labrat011/google-autocomplete-scraper"}],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/google.com.json"},{"schema":"ratradar.site/v1","host":"www.amazon.com","name":"Amazon","category":"shop","country":"United States","verdict":"hostile","hostility_score":60,"recommended_approach":"real_browser","steps":["Use a rendering browser, slow requests, and rotating proxies.","A browser load called JSON endpoints that look like data, for example www.amazon.com/hz/rhf. Untested, check the terms before using them."],"risks":["Hostile to automation. Keep volumes low and cache.","robots.txt forbids 11 of 14 known AI crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":false,"headless_browser":true,"needs_proxies":true,"official_api_advised":false},"protections":{"vendors":["AWS WAF","CloudFront"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"CloudFront"},"robots":{"found":true,"disallow_all":false,"disallow_rules":120,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","Claude-Web","CCBot","PerplexityBot","Google-Extended","Bytespider","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":11,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.amazon.com/hz/rhf","www.amazon.com/dram/renderLazyLoadedStreaming","www.amazon.com/hokuto/suggestions"],"inner_page":null},"performance":{"response_ms":5416,"bytes":2007,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"A challenge page appeared","points":45},{"label":"AWS WAF detected","points":12},{"label":"CloudFront detected","points":2},{"label":"Slow response (5416 ms)","points":5},{"label":"A real browser got in though a plain request did not","points":0}],"eu_ai_training":{"level":"strong","signals":[{"kind":"robots_txt","label":"robots.txt blocks 11 AI crawlers"},{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":202,"headless_browser_status":202,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/amazon.com.json"},{"schema":"ratradar.site/v1","host":"www.linkedin.com","name":"LinkedIn","category":"social","country":"United States","verdict":"hostile","hostility_score":50,"recommended_approach":"official_api_or_skip","steps":["The home page is a login wall, so most of the data needs an account. Scraping behind a login usually breaks the terms. Use an official API, or only the public pages that load without a login."],"risks":["Logged-in scraping risks account bans and terms-of-service claims.","robots.txt forbids 10 of 14 known AI crawlers (GPTBot, ChatGPT-User, ClaudeBot, Claude-Web, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":false,"headless_browser":true,"needs_proxies":false,"official_api_advised":true},"protections":{"vendors":["Cloudflare","PerimeterX"],"challenge_page":false,"javascript_shell":false,"login_wall":true,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","ClaudeBot","Claude-Web","anthropic-ai","CCBot","PerplexityBot","Google-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":10,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":5532,"bytes":140762,"redirects":0,"cookies_set":6},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"PerimeterX seen by the browser","points":20},{"label":"Slow response (5532 ms)","points":5},{"label":"Raised to at least 50 because the home page is a login wall","points":0}],"eu_ai_training":{"level":"strong","signals":[{"kind":"robots_txt","label":"robots.txt blocks 10 AI crawlers"},{"kind":"login_wall","label":"a login wall"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[{"slug":"linkedin-jobs-scraper","title":"LinkedIn Jobs Scraper","url":"https://apify.com/labrat011/linkedin-jobs-scraper"}],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/linkedin.com.json"},{"schema":"ratradar.site/v1","host":"www.facebook.com","name":"Facebook","category":"social","country":"United States","verdict":"hostile","hostility_score":50,"recommended_approach":"respect_robots_skip","steps":["robots.txt disallows every crawler. Do not scrape it. Look for an official API, a data license, or a partner feed."],"risks":["Scraping against an explicit robots.txt ban is the quickest way to a legal or blocking problem."],"can_scrape":{"plain_http":false,"headless_browser":true,"needs_proxies":false,"official_api_advised":true},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":true,"disallow_rules":1,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.facebook.com/sitemap/cpg_offplat_future_public_events_sitemap.xml.gz","https://www.facebook.com/sitemap/public_active_groups_sitemap.xml.gz","https://www.facebook.com/sitemap/groups_popular_posts_sitemap_qe.xml.gz"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":5669,"bytes":400000,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15},{"label":"robots.txt disallows everything","points":10},{"label":"Slow response (5669 ms)","points":5},{"label":"Raised to at least 50 because robots.txt disallows everything","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt bans every crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/facebook.com.json"},{"schema":"ratradar.site/v1","host":"www.instagram.com","name":"Instagram","category":"social","country":"United States","verdict":"hostile","hostility_score":50,"recommended_approach":"respect_robots_skip","steps":["robots.txt disallows every crawler. Do not scrape it. Look for an official API, a data license, or a partner feed."],"risks":["Scraping against an explicit robots.txt ban is the quickest way to a legal or blocking problem."],"can_scrape":{"plain_http":false,"headless_browser":true,"needs_proxies":false,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":true,"disallow_rules":1,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.instagram.com/sitemap/ig_places_sitemap.xml.gz","https://www.instagram.com/sitemap/ig_seo_profile_sitemap.xml.gz","https://www.instagram.com/sitemap/ig_seo_profile_sitemap_non_media_eligible.xml.gz"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":5876,"bytes":400000,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"robots.txt disallows everything","points":10},{"label":"Slow response (5876 ms)","points":5},{"label":"Raised to at least 50 because robots.txt disallows everything","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt bans every crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/instagram.com.json"},{"schema":"ratradar.site/v1","host":"x.com","name":"X","category":"social","country":"United States","verdict":"hostile","hostility_score":50,"recommended_approach":"respect_robots_skip","steps":["robots.txt disallows every crawler. Do not scrape it. Look for an official API, a data license, or a partner feed.","robots.txt asks for a crawl delay of 1 seconds."],"risks":["Scraping against an explicit robots.txt ban is the quickest way to a legal or blocking problem."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":false,"official_api_advised":true},"protections":{"vendors":["Cloudflare","Fastly","CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare envoy"},"robots":{"found":true,"disallow_all":true,"disallow_rules":2,"crawl_delay_seconds":1,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://x.com/sitemap.xml"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":5455,"bytes":42239,"redirects":0,"cookies_set":7},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"Fastly detected","points":2},{"label":"CloudFront detected","points":2},{"label":"robots.txt disallows everything","points":10},{"label":"Slow response (5455 ms)","points":5},{"label":"Raised to at least 50 because robots.txt disallows everything","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt bans every crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":0,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/x.com.json"},{"schema":"ratradar.site/v1","host":"www.reddit.com","name":"Reddit","category":"social","country":"United States","verdict":"hostile","hostility_score":50,"recommended_approach":"respect_robots_skip","steps":["robots.txt disallows every crawler. Do not scrape it. Look for an official API, a data license, or a partner feed."],"risks":["Scraping against an explicit robots.txt ban is the quickest way to a legal or blocking problem."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":false,"official_api_advised":true},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":["retry-after"],"cdn":"","server":"snooserv"},"robots":{"found":true,"disallow_all":true,"disallow_rules":1,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":5465,"bytes":8397,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"robots.txt disallows everything","points":10},{"label":"Slow response (5465 ms)","points":5},{"label":"Raised to at least 50 because robots.txt disallows everything","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt bans every crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/reddit.com.json"},{"schema":"ratradar.site/v1","host":"github.com","name":"GitHub","category":"dev","country":"United States","verdict":"open","hostility_score":5,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt."],"risks":["robots.txt forbids some AI crawlers: Bytespider."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"GitHub","server":"github.com"},"robots":{"found":true,"disallow_all":false,"disallow_rules":180,"crawl_delay_seconds":null,"ai_crawlers_blocked":["Bytespider"],"ai_crawlers_blocked_count":1,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":true,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":5737,"bytes":400000,"redirects":0,"cookies_set":3},"why_this_score":[{"label":"Slow response (5737 ms)","points":5}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 1 AI crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/github.com.json"},{"schema":"ratradar.site/v1","host":"en.wikipedia.org","name":"Wikipedia","category":"dev","country":"United States","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Read the JSON-LD on the page (Article). It is cleaner than the markup.","The site publishes 4 RSS or Atom feeds. Prefer them for new items."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"mw-web.codfw.main-6b979c7866-ghfz7"},"robots":{"found":true,"disallow_all":false,"disallow_rules":428,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://en.wikipedia.org/w/rest.php/site/v1/sitemap/0"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":["Article"],"feeds":4,"feed_urls":["https://en.wikipedia.org/w/api.php?action=featuredfeed&feed=potd&feedformat=atom","https://en.wikipedia.org/w/api.php?action=featuredfeed&feed=featured&feedformat=atom","https://en.wikipedia.org/w/api.php?action=featuredfeed&feed=onthisday&feedformat=atom"],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":177,"bytes":253159,"redirects":1,"cookies_set":6},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/en.wikipedia.org.json"},{"schema":"ratradar.site/v1","host":"news.ycombinator.com","name":"Hacker News","category":"dev","country":"United States","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","robots.txt asks for a crawl delay of 30 seconds.","The site publishes 1 RSS or Atom feed. Prefer them for new items."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":12,"crawl_delay_seconds":30,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":1,"feed_urls":["https://news.ycombinator.com/rss"],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":52,"bytes":34273,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/news.ycombinator.com.json"},{"schema":"ratradar.site/v1","host":"stackoverflow.com","name":"Stack Overflow","category":"dev","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":116,"bytes":5188,"redirects":1,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/stackoverflow.com.json"},{"schema":"ratradar.site/v1","host":"www.nytimes.com","name":"NYTimes","category":"news","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies.","Enumerate pages from the sitemap: https://www.nytimes.com/sitemaps/new/news.xml.gz"],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans.","robots.txt forbids 13 of 14 known AI crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["DataDome","Fastly"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":"DataDome"},"robots":{"found":true,"disallow_all":false,"disallow_rules":86,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","Claude-Web","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Bytespider","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":13,"sitemap_urls":["https://www.nytimes.com/sitemaps/new/news.xml.gz","https://www.nytimes.com/sitemaps/new/sitemap.xml.gz","https://www.nytimes.com/sitemaps/new/collections.xml.gz"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":49,"bytes":771,"redirects":0,"cookies_set":9},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"DataDome detected","points":20},{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"strong","signals":[{"kind":"robots_txt","label":"robots.txt blocks 13 AI crawlers"},{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/nytimes.com.json"},{"schema":"ratradar.site/v1","host":"www.zillow.com","name":"Zillow","category":"property","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Enumerate pages from the sitemap: https://mortgageapi.zillow.com/sitemap.xml"],"risks":["Hostile to automation. Keep volumes low and cache.","robots.txt forbids 3 of 14 known AI crawlers (Amazonbot, Applebot-Extended, Meta-ExternalAgent). An agent that identifies as an AI crawler should not fetch this site.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront","PerimeterX"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"CloudFront"},"robots":{"found":true,"disallow_all":false,"disallow_rules":210,"crawl_delay_seconds":null,"ai_crawlers_blocked":["Amazonbot","Applebot-Extended","Meta-ExternalAgent"],"ai_crawlers_blocked_count":3,"sitemap_urls":["https://mortgageapi.zillow.com/sitemap.xml","https://www.zillow.com/xml/indexes/us/static.xml.gz","https://www.zillow.com/xml/indexes/us/hdp/for-sale-by-agent.xml.gz"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.zillow.com/homedetails/123-W-14th-St-Crowley-LA-70526/252233700_zpid/","status":200,"json_ld_types":["RealEstateListing","Product"],"skipped":null,"sitemap":"https://www.zillow.com/xml/sitemaps/us/hdp/other/latest.xml.gz"}},"performance":{"response_ms":489,"bytes":400000,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"CloudFront detected","points":2},{"label":"PerimeterX seen by the browser","points":20},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"strong","signals":[{"kind":"robots_txt","label":"robots.txt blocks 3 AI crawlers"},{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/zillow.com.json"},{"schema":"ratradar.site/v1","host":"www.ticketmaster.com","name":"Ticketmaster","category":"market","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Read the JSON-LD on the page (WebSite, Organization). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.ticketmaster.com/epsf/bff70e60/asset/block.json. Untested, check the terms before using them."],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":70,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":["WebSite","Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.ticketmaster.com/epsf/bff70e60/asset/block.json"],"inner_page":null},"performance":{"response_ms":796,"bytes":400000,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"Fastly detected","points":2},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/ticketmaster.com.json"},{"schema":"ratradar.site/v1","host":"www.indeed.com","name":"Indeed","category":"jobs","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":169,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":33,"bytes":27581,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/indeed.com.json"},{"schema":"ratradar.site/v1","host":"www.glassdoor.com","name":"Glassdoor","category":"jobs","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans.","robots.txt forbids 10 of 14 known AI crawlers (GPTBot, ClaudeBot, Claude-Web, anthropic-ai, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":141,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ClaudeBot","Claude-Web","anthropic-ai","CCBot","Google-Extended","Bytespider","Amazonbot","Applebot-Extended","cohere-ai"],"ai_crawlers_blocked_count":10,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":true,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":28,"bytes":24777,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"strong","signals":[{"kind":"robots_txt","label":"robots.txt blocks 10 AI crawlers"},{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/glassdoor.com.json"},{"schema":"ratradar.site/v1","host":"www.yelp.com","name":"Yelp","category":"reviews","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"respect_robots_skip","steps":["robots.txt disallows every crawler. Do not scrape it. Look for an official API, a data license, or a partner feed."],"risks":["Scraping against an explicit robots.txt ban is the quickest way to a legal or blocking problem.","robots.txt forbids 10 of 14 known AI crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":false,"official_api_advised":true},"protections":{"vendors":["DataDome","Fastly"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":"DataDome"},"robots":{"found":true,"disallow_all":true,"disallow_rules":1,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","Claude-Web","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Meta-ExternalAgent"],"ai_crawlers_blocked_count":10,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":65,"bytes":778,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"DataDome detected","points":20},{"label":"Fastly detected","points":2},{"label":"robots.txt disallows everything","points":10},{"label":"Raised to at least 50 because robots.txt disallows everything","points":0}],"eu_ai_training":{"level":"strong","signals":[{"kind":"robots_txt","label":"robots.txt bans every crawler"},{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/yelp.com.json"},{"schema":"ratradar.site/v1","host":"www.trustpilot.com","name":"Trustpilot","category":"reviews","country":"Denmark","verdict":"blocked","hostility_score":94,"recommended_approach":"respect_robots_skip","steps":["robots.txt disallows every crawler. Do not scrape it. Look for an official API, a data license, or a partner feed.","Enumerate pages from the sitemap: https://sitemaps.trustpilot.com/index_en-us.xml","A browser load called JSON endpoints that look like data, for example www.trustpilot.com/api/consumersitesearch-api/businessunits/semantic-suggestions. Untested, check the terms before using them."],"risks":["Scraping against an explicit robots.txt ban is the quickest way to a legal or blocking problem.","robots.txt forbids 4 of 14 known AI crawlers (anthropic-ai, Bytespider, Applebot-Extended, cohere-ai). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":false,"official_api_advised":true},"protections":{"vendors":["AWS WAF","CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"CloudFront"},"robots":{"found":true,"disallow_all":true,"disallow_rules":1,"crawl_delay_seconds":null,"ai_crawlers_blocked":["anthropic-ai","Bytespider","Applebot-Extended","cohere-ai"],"ai_crawlers_blocked_count":4,"sitemap_urls":["https://sitemaps.trustpilot.com/index_en-us.xml","https://trustpilot.com/trust/sitemaps/domain_en-us.xml","https://trustpilot.com/blog/sitemaps/domain_en-us.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.trustpilot.com/api/consumersitesearch-api/businessunits/semantic-suggestions"],"inner_page":null},"performance":{"response_ms":34,"bytes":991,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"AWS WAF detected","points":12},{"label":"CloudFront detected","points":2},{"label":"robots.txt disallows everything","points":10},{"label":"Raised to at least 50 because robots.txt disallows everything","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt bans every crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/trustpilot.com.json"},{"schema":"ratradar.site/v1","host":"www.airbnb.com","name":"Airbnb","category":"travel","country":"United States","verdict":"guarded","hostility_score":30,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429.","Read the JSON-LD on the page (WebSite). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.airbnb.com/api/v2/get-data-layer-variables. Untested, check the terms before using them."],"risks":["robots.txt forbids 3 of 14 known AI crawlers (GPTBot, ClaudeBot, Applebot-Extended). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai","DataDome"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":61,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ClaudeBot","Applebot-Extended"],"ai_crawlers_blocked_count":3,"sitemap_urls":["https://www.airbnb.com/sitemap-master-index.xml.gz"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":["WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.airbnb.com/api/v2/get-data-layer-variables","d0a7e.airbnb.com/js/","www.airbnb.com/api/v3/NaviServerAnnouncementsQuery/060ac9ab481520a361f3efb91bb92d04e43b30d27f81e3a4eabb1e088c08e0f5","www.airbnb.com/api/v3/PromotionsByPlacementsQuery/dc003b55765e5bbee859cd3b605bdf4facd442195c950f07e01dfc56ae5ae8d8","www.airbnb.com/api/v2/client_configs"],"inner_page":null},"performance":{"response_ms":539,"bytes":400000,"redirects":0,"cookies_set":7},"why_this_score":[{"label":"Akamai detected","points":10},{"label":"DataDome seen by the browser","points":20}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 3 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/airbnb.com.json"},{"schema":"ratradar.site/v1","host":"www.booking.com","name":"Booking","category":"travel","country":"Netherlands","verdict":"hostile","hostility_score":59,"recommended_approach":"real_browser","steps":["Use a rendering browser, slow requests, and rotating proxies.","Enumerate pages from the sitemap: https://www.booking.com/sitembk-airport-index.xml"],"risks":["Hostile to automation. Keep volumes low and cache."],"can_scrape":{"plain_http":false,"headless_browser":true,"needs_proxies":true,"official_api_advised":false},"protections":{"vendors":["AWS WAF","CloudFront"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"CloudFront"},"robots":{"found":true,"disallow_all":false,"disallow_rules":77,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.booking.com/sitembk-airport-index.xml","https://www.booking.com/sitembk-attractions-category-city-index.xml","https://www.booking.com/sitembk-attractions-category-region-index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.booking.com/city/gb/barmer.he.html","status":200,"json_ld_types":["WebSite","Corporation","Brand","ImageObject","Person","CollectionPage","City"],"skipped":null,"sitemap":"https://www.booking.com/sitembk-city-he.0006.xml.gz"}},"performance":{"response_ms":74,"bytes":8410,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"A challenge page appeared","points":45},{"label":"AWS WAF detected","points":12},{"label":"CloudFront detected","points":2},{"label":"A real browser got in though a plain request did not","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":202,"headless_browser_status":202,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/booking.com.json"},{"schema":"ratradar.site/v1","host":"www.ebay.com","name":"eBay","category":"market","country":"United States","verdict":"blocked","hostility_score":80,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies.","Enumerate pages from the sitemap: https://www.ebay.com/lst/AUCTION-0-index.xml"],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans.","robots.txt forbids 9 of 14 known AI crawlers (GPTBot, ClaudeBot, anthropic-ai, CCBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"AkamaiGHost"},"robots":{"found":true,"disallow_all":false,"disallow_rules":216,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ClaudeBot","anthropic-ai","CCBot","PerplexityBot","Bytespider","Amazonbot","Applebot-Extended","Meta-ExternalAgent"],"ai_crawlers_blocked_count":9,"sitemap_urls":["https://www.ebay.com/lst/AUCTION-0-index.xml","https://www.ebay.com/lst/BROWSE-0-index.xml","https://www.ebay.com/lst/KWSRP-0-index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":65,"bytes":1831,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 9 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/ebay.com.json"},{"schema":"ratradar.site/v1","host":"www.walmart.com","name":"Walmart","category":"shop","country":"United States","verdict":"open","hostility_score":20,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.walmart.com/sitemap_category.xml","Read the JSON-LD on the page (Organization, WebSite). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.walmart.com/orchestra/home/graphql/Location/26e4ca8c866334908ce825a5749f6aa13f772b98a527b0d0b7fe457818c5b349. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["PerimeterX"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":57,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.walmart.com/sitemap_category.xml","https://www.walmart.com/sitemap_store_main.xml","https://www.walmart.com/help/sitemap_gm.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["Organization","WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.walmart.com/orchestra/home/graphql/Location/26e4ca8c866334908ce825a5749f6aa13f772b98a527b0d0b7fe457818c5b349"],"inner_page":{"url":"https://www.walmart.com/ip/Mens-4-5-oz-SoftStyle-T-Shirt-10-Pack/399260270","status":200,"json_ld_types":["WebPage"],"skipped":null,"sitemap":"https://www.walmart.com/sitemap_product_03_1.xml.gz"}},"performance":{"response_ms":88,"bytes":336891,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"PerimeterX detected","points":20}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/walmart.com.json"},{"schema":"ratradar.site/v1","host":"www.target.com","name":"Target","category":"shop","country":"United States","verdict":"open","hostility_score":20,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.target.com/sitemap_keywords-index.xml.gz","Read the JSON-LD on the page (WebPage). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example redoak.target.com/content-publish/pages/v1. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["PerimeterX"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":114,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.target.com/sitemap_keywords-index.xml.gz","https://www.target.com/sitemap_stores-index.xml.gz","https://www.target.com/sitemap_taxonomy-categories-index.xml.gz"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["WebPage"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["redoak.target.com/content-publish/pages/v1","api.target.com/firefly_events/v1/events/experiment_exposed","api.target.com/firefly_events/v1/events/traffic_source","api.target.com/firefly_events/v1/events/cdui_page_view"],"inner_page":{"url":"https://www.target.com/p/campus-lab-university-of-arkansas-at-monticello-official-uam-boll-weevils-logo-unisex-adult-pull-over-hoodie/-/A-1000000076","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.target.com/pdp/sitemap_00-0001.xml.gz"}},"performance":{"response_ms":611,"bytes":400000,"redirects":0,"cookies_set":13},"why_this_score":[{"label":"PerimeterX detected","points":20}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/target.com.json"},{"schema":"ratradar.site/v1","host":"www.etsy.com","name":"Etsy","category":"market","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["DataDome","Fastly"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":"DataDome"},"robots":{"found":true,"disallow_all":false,"disallow_rules":1650,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":true,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":28,"bytes":776,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"DataDome detected","points":20},{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/etsy.com.json"},{"schema":"ratradar.site/v1","host":"www.shopify.com","name":"Shopify","category":"shop","country":"Canada","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.shopify.com/sitemaps_list.xml","Read the JSON-LD on the page (Corporation). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.shopify.com/.well-known/dux. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":93,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.shopify.com/sitemaps_list.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["Corporation"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.shopify.com/.well-known/dux"],"inner_page":{"url":"https://www.shopify.com/news/shopify-q3-2026-financial-advisory","status":200,"json_ld_types":["NewsArticle"],"skipped":null,"sitemap":"https://www.shopify.com/sitemap_news.xml"}},"performance":{"response_ms":83,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/shopify.com.json"},{"schema":"ratradar.site/v1","host":"www.cloudflare.com","name":"Cloudflare","category":"dev","country":"United States","verdict":"open","hostility_score":8,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.cloudflare.com/sitemap.xml","Read the JSON-LD on the page (Organization, WebSite, WebPage). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.cloudflare.com/static/z/t. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare","Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.cloudflare.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["Organization","WebSite","WebPage"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.cloudflare.com/static/z/t","api.company-target.com/api/v3/ip.json"],"inner_page":{"url":"https://www.cloudflare.com/products/artifacts","status":200,"json_ld_types":["Organization","WebSite","WebPage"],"skipped":null,"sitemap":"https://www.cloudflare.com/sitemap.xml"}},"performance":{"response_ms":172,"bytes":400000,"redirects":0,"cookies_set":5},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/cloudflare.com.json"},{"schema":"ratradar.site/v1","host":"openai.com","name":"OpenAI","category":"ai","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Enumerate pages from the sitemap: https://openai.com/sitemap.xml"],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":1,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://openai.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://openai.com/news/product-releases/","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://openai.com/sitemap.xml/product/"}},"performance":{"response_ms":177,"bytes":400000,"redirects":0,"cookies_set":5},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/openai.com.json"},{"schema":"ratradar.site/v1","host":"www.anthropic.com","name":"Anthropic","category":"ai","country":"United States","verdict":"open","hostility_score":8,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.anthropic.com/sitemap.xml","A browser load called JSON endpoints that look like data, for example cdn.prod.website-files.com/67ce28cfec624e2b733f8a52/67d47b4c03b69d41f28cc15c_logo-lottie.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare","CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.anthropic.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["cdn.prod.website-files.com/67ce28cfec624e2b733f8a52/67d47b4c03b69d41f28cc15c_logo-lottie.json","www.anthropic.com/api/country","hubspotonwebflow.com/assets/js/blockedDomains.json"],"inner_page":{"url":"https://www.anthropic.com/news/anthropic-achieves-iso-42001-certification-for-responsible-ai","status":200,"json_ld_types":["NewsArticle"],"skipped":null,"sitemap":"https://www.anthropic.com/sitemap.xml"}},"performance":{"response_ms":100,"bytes":201364,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/anthropic.com.json"},{"schema":"ratradar.site/v1","host":"www.sec.gov","name":"SEC EDGAR","category":"gov","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Enumerate pages from the sitemap: https://www.sec.gov/sec-sitemap.xml"],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":48,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.sec.gov/sec-sitemap.xml","https://www.sec.gov/sitemap/sitemap-index.xml","https://www.investor.gov/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.sec.gov/foia/iareports/ia050117.zip","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.sec.gov/sec-sitemap.xml?page=1"}},"performance":{"response_ms":128,"bytes":225487,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"Akamai detected","points":10},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/sec.gov.json"},{"schema":"ratradar.site/v1","host":"pubmed.ncbi.nlm.nih.gov","name":"PubMed","category":"gov","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","robots.txt asks for a crawl delay of 1 seconds.","Enumerate pages from the sitemap: https://pubmed.ncbi.nlm.nih.gov/sitemap?p=index.xml"],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":45,"crawl_delay_seconds":1,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://pubmed.ncbi.nlm.nih.gov/sitemap?p=index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":194,"bytes":42284,"redirects":0,"cookies_set":3},"why_this_score":[{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[{"slug":"pubmed-scraper","title":"PubMed Scraper","url":"https://apify.com/labrat011/pubmed-scraper"}],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/pubmed.ncbi.nlm.nih.gov.json"},{"schema":"ratradar.site/v1","host":"clinicaltrials.gov","name":"ClinicalTrials","category":"gov","country":"United States","verdict":"open","hostility_score":15,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls.","robots.txt asks for a crawl delay of 1 seconds.","Enumerate pages from the sitemap: https://clinicaltrials.gov/assets/sitemap.xml","A browser load called JSON endpoints that look like data, for example clinicaltrials.gov/api/int/suggest. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"istio-envoy"},"robots":{"found":true,"disallow_all":false,"disallow_rules":3,"crawl_delay_seconds":1,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://clinicaltrials.gov/assets/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["clinicaltrials.gov/api/int/suggest","clinicaltrials.gov/api/int/stats/report","cdn.ncbi.nlm.nih.gov/ncbi-media/alerts/alerts.json"],"inner_page":{"url":"https://clinicaltrials.gov/api/seo/studies","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://clinicaltrials.gov/api/seo"}},"performance":{"response_ms":203,"bytes":94247,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[{"slug":"clinical-trials-scraper","title":"Clinical Trials Scraper","url":"https://apify.com/labrat011/clinical-trials-scraper"}],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/clinicaltrials.gov.json"},{"schema":"ratradar.site/v1","host":"www.fda.gov","name":"FDA","category":"gov","country":"United States","verdict":"open","hostility_score":10,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","robots.txt asks for a crawl delay of 30 seconds.","Enumerate pages from the sitemap: https://www.fda.gov/sitemap.xml","A browser load called JSON endpoints that look like data, for example www.fda.gov/W7MZ2/ZDG_2/-2HP/IC0_/ea/1OYwVVmJVcpJ6tYE/SwNsVyoB/On5QP3/crezo. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":25,"crawl_delay_seconds":30,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.fda.gov/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.fda.gov/W7MZ2/ZDG_2/-2HP/IC0_/ea/1OYwVVmJVcpJ6tYE/SwNsVyoB/On5QP3/crezo"],"inner_page":{"url":"https://www.fda.gov/media/69866/download","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.fda.gov/sitemap.xml?page=1"}},"performance":{"response_ms":61,"bytes":41075,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[{"slug":"fda-drug-labels-scraper","title":"FDA  Drug Labels Scraper","url":"https://apify.com/labrat011/fda-drug-labels-scraper"}],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/fda.gov.json"},{"schema":"ratradar.site/v1","host":"play.google.com","name":"Google Play","category":"dev","country":"United States","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://play.google.com/sitemaps/sitemaps-index-0.xml","A browser load called JSON endpoints that look like data, for example play.google.com/_/PlayStoreUi/data/batchexecute. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Google","server":"ESF"},"robots":{"found":true,"disallow_all":false,"disallow_rules":97,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://play.google.com/sitemaps/sitemaps-index-0.xml","https://play.google.com/sitemaps/sitemaps-index-1.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["play.google.com/_/PlayStoreUi/data/batchexecute","ssl.gstatic.com/support/realtime/operatorParams"],"inner_page":{"url":"https://play.google.com/store/books/container/promotion_1002302_msb_exc_id_bo","status":404,"json_ld_types":[],"skipped":null,"sitemap":"https://play.google.com/sitemaps/play_sitemaps_2026-10-10_1791637317-00000-of-85970.xml.gz"}},"performance":{"response_ms":278,"bytes":400000,"redirects":2,"cookies_set":1},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[{"slug":"google-play-scraper","title":"Google Play Store Scraper: Apps, Reviews & Top Charts","url":"https://apify.com/labrat011/google-play-scraper"}],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/play.google.com.json"},{"schema":"ratradar.site/v1","host":"apps.apple.com","name":"App Store","category":"dev","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://apps.apple.com/sitemaps_apps_index_chart_1.xml","Read the JSON-LD on the page (Organization). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example xp.apple.com/config/1/report/xp_amp_appstore_perf. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":"daiquiri/5"},"robots":{"found":true,"disallow_all":false,"disallow_rules":5,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://apps.apple.com/sitemaps_apps_index_chart_1.xml","https://apps.apple.com/sitemaps_apps_index_app_1.xml","https://apps.apple.com/sitemaps_apps_index_new-app_1.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["xp.apple.com/config/1/report/xp_amp_appstore_perf","xp.apple.com/config/1/report/xp_amp_appstore_unidentified"],"inner_page":{"url":"https://apps.apple.com/br/app/desafio-75-dias-better/id6667095733","status":200,"json_ld_types":["Organization","SoftwareApplication","BreadcrumbList"],"skipped":null,"sitemap":"https://apps.apple.com/sitemaps_apps_app_4_3.xml.gz"}},"performance":{"response_ms":149,"bytes":400000,"redirects":1,"cookies_set":1},"why_this_score":[{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[{"slug":"app-store-scraper","title":"Apple App Store Scraper","url":"https://apify.com/labrat011/app-store-scraper"}],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/apps.apple.com.json"},{"schema":"ratradar.site/v1","host":"telegram.org","name":"Telegram","category":"social","country":"United Arab Emirates","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"nginx/1.30.1"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":667,"bytes":19928,"redirects":0,"cookies_set":1},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[{"slug":"telegram-channel-scraper","title":"Telegram Channel Scraper","url":"https://apify.com/labrat011/telegram-channel-scraper"}],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/telegram.org.json"},{"schema":"ratradar.site/v1","host":"substack.com","name":"Substack","category":"news","country":"United States","verdict":"open","hostility_score":8,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://substack.com/sitemap.xml","A browser load called JSON endpoints that look like data, for example substack.com/api/v1/messages/unread-count. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare","Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":14,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://substack.com/sitemap.xml","https://substack.com/news_sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["substack.com/api/v1/messages/unread-count","substack.com/api/v1/search-modules","substack.com/api/v1/reader/feed/tabs"],"inner_page":null},"performance":{"response_ms":441,"bytes":306875,"redirects":0,"cookies_set":9},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[{"slug":"substack-scraper","title":"Substack Scraper","url":"https://apify.com/labrat011/substack-scraper"}],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/substack.com.json"},{"schema":"ratradar.site/v1","host":"www.bbc.com","name":"BBC","category":"news","country":"United Kingdom","verdict":"open","hostility_score":8,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.bbc.com/afrique/sitemap.xml","Read the JSON-LD on the page (WebPage). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.bbc.co.uk/userinfo. Untested, check the terms before using them."],"risks":["robots.txt forbids 14 of 14 known AI crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly","Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":"BBC-GTM"},"robots":{"found":true,"disallow_all":false,"disallow_rules":49,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","Claude-Web","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Bytespider","Amazonbot","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":14,"sitemap_urls":["https://www.bbc.com/afrique/sitemap.xml","https://www.bbc.com/arabic/sitemap.xml","https://www.bbc.com/bbcx/audio_archive_sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebPage"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.bbc.co.uk/userinfo","web-cdn.api.bbci.co.uk/xd/schedule"],"inner_page":{"url":"https://www.bbc.com/sport/football/articles/cd94713djgyo","status":200,"json_ld_types":["Article"],"skipped":null,"sitemap":"https://www.bbc.com/sitemaps/https-sitemap-com-news-1.xml"}},"performance":{"response_ms":125,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Fastly detected","points":2},{"label":"Cloudflare seen by the browser","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 14 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/bbc.com.json"},{"schema":"ratradar.site/v1","host":"www.theguardian.com","name":"The Guardian","category":"news","country":"United Kingdom","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: http://www.theguardian.com/sitemaps/news.xml"],"risks":["robots.txt forbids 8 of 14 known AI crawlers (ClaudeBot, anthropic-ai, CCBot, PerplexityBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":48,"crawl_delay_seconds":null,"ai_crawlers_blocked":["ClaudeBot","anthropic-ai","CCBot","PerplexityBot","Bytespider","Amazonbot","Applebot-Extended","Meta-ExternalAgent"],"ai_crawlers_blocked_count":8,"sitemap_urls":["http://www.theguardian.com/sitemaps/news.xml","http://www.theguardian.com/sitemaps/video.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.theguardian.com/news/gallery/2026/oct/11/marge-simpson-rides-the-subway-and-ali-g-returns-photos-of-the-weekend","status":200,"json_ld_types":["NewsArticle","WebPage"],"skipped":null,"sitemap":"http://www.theguardian.com/sitemaps/news.xml"}},"performance":{"response_ms":559,"bytes":400000,"redirects":1,"cookies_set":7},"why_this_score":[],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 8 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/theguardian.com.json"},{"schema":"ratradar.site/v1","host":"www.spiegel.de","name":"Spiegel","category":"news","country":"Germany","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.spiegel.de/sitemaps/news-de.xml","Read the JSON-LD on the page (WebPage). It is cleaner than the markup.","The site publishes 2 RSS or Atom feeds. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example cdn.prod.www.spiegel.de/assets/news/breakingnews.json. Untested, check the terms before using them."],"risks":["robots.txt forbids 12 of 14 known AI crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":1,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","Claude-Web","anthropic-ai","CCBot","Bytespider","Amazonbot","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":12,"sitemap_urls":["https://www.spiegel.de/sitemaps/news-de.xml","https://www.spiegel.de/sitemaps/videos/sitemap.xml","https://www.spiegel.de/plus/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebPage"],"feeds":2,"feed_urls":["https://www.spiegel.de/schlagzeilen/index.rss","https://www.spiegel.de/index.rss"],"api_endpoints_seen":["cdn.prod.www.spiegel.de/assets/news/breakingnews.json","sp-spiegel-de.spiegel.de/wrapper/v2/meta-data","sp-spiegel-de.spiegel.de/wrapper/v2/messages","sams.spiegel.de/ee/v1/identity/acquire","sams.spiegel.de/ee/irl1/v1/interact"],"inner_page":{"url":"https://www.spiegel.de/wirtschaft/unternehmen/staatsverschuldung-in-europa-haben-wir-aus-der-letzten-eurokrise-nichts-gelernt-a-8baca41f-f842-4b1e-af99-b92956f4491e","status":200,"json_ld_types":["NewsArticle","BreadcrumbList"],"skipped":null,"sitemap":"https://www.spiegel.de/sitemaps/news-de.xml"}},"performance":{"response_ms":1218,"bytes":400000,"redirects":0,"cookies_set":1},"why_this_score":[],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 12 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/spiegel.de.json"},{"schema":"ratradar.site/v1","host":"www.lemonde.fr","name":"Le Monde","category":"news","country":"France","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.lemonde.fr/sitemap_news.xml","Read the JSON-LD on the page (NewsMediaOrganization). It is cleaner than the markup.","The site publishes 1 RSS or Atom feed. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example cmp.lemonde.fr/html/lemonde-en/standard-transparent-deny.html. Untested, check the terms before using them."],"risks":["robots.txt forbids 9 of 14 known AI crawlers (ClaudeBot, Claude-Web, anthropic-ai, CCBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":30,"crawl_delay_seconds":null,"ai_crawlers_blocked":["ClaudeBot","Claude-Web","anthropic-ai","CCBot","Google-Extended","Bytespider","Amazonbot","Applebot-Extended","cohere-ai"],"ai_crawlers_blocked_count":9,"sitemap_urls":["https://www.lemonde.fr/sitemap_news.xml","https://www.lemonde.fr/sitemap_index.xml","https://www.lemonde.fr/en/sitemap_news.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["NewsMediaOrganization"],"feeds":1,"feed_urls":["https://www.lemonde.fr/en/rss/une.xml"],"api_endpoints_seen":["cmp.lemonde.fr/html/lemonde-en/standard-transparent-deny.html","cmp.lemonde.fr/api/impression","campaign.services.forecast.sirius.press/forecast.campaigns.v1.EvaluateService/EvaluateCampaignBulk"],"inner_page":{"url":"https://www.lemonde.fr/sport/article/2026/10/11/tennis-invitee-par-les-organisateurs-la-chinoise-zheng-qinwen-remporte-le-wta-1000-de-pekin_6791927_3242.html","status":200,"json_ld_types":["NewsArticle","BreadcrumbList"],"skipped":null,"sitemap":"https://www.lemonde.fr/sitemap_news.xml"}},"performance":{"response_ms":203,"bytes":221582,"redirects":1,"cookies_set":2},"why_this_score":[{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 9 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/lemonde.fr.json"},{"schema":"ratradar.site/v1","host":"yandex.com","name":"Yandex","category":"search","country":"Russia","verdict":"open","hostility_score":15,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls.","Enumerate pages from the sitemap: https://yandex.com/support/sitemap.xml","A browser load called JSON endpoints that look like data, for example yandex.com/suggest/suggest-ya.cgi. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":221,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://yandex.com/support/sitemap.xml","https://yandex.com/blog/sitemap.xml","https://yandex.com/turbo/public/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["yandex.com/suggest/suggest-ya.cgi","mc.yandex.com/watch/3/1"],"inner_page":{"url":"https://yandex.com/support/id/troubleshooting/","status":404,"json_ld_types":[],"skipped":null,"sitemap":"https://yandex.com/support/sitemap.xml"}},"performance":{"response_ms":1867,"bytes":386486,"redirects":0,"cookies_set":10},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/yandex.com.json"},{"schema":"ratradar.site/v1","host":"www.baidu.com","name":"Baidu","category":"search","country":"China","verdict":"hostile","hostility_score":50,"recommended_approach":"respect_robots_skip","steps":["robots.txt disallows every crawler. Do not scrape it. Look for an official API, a data license, or a partner feed."],"risks":["Scraping against an explicit robots.txt ban is the quickest way to a legal or blocking problem."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":false,"official_api_advised":true},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"BWS/1.1"},"robots":{"found":true,"disallow_all":true,"disallow_rules":1,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":1436,"bytes":400000,"redirects":0,"cookies_set":6},"why_this_score":[{"label":"robots.txt disallows everything","points":10},{"label":"Raised to at least 50 because robots.txt disallows everything","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt bans every crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":0,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/baidu.com.json"},{"schema":"ratradar.site/v1","host":"www.alibaba.com","name":"Alibaba","category":"shop","country":"China","verdict":"open","hostility_score":10,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Read the JSON-LD on the page (WebSite). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example login.alibaba.com/getEnvironment.do. Untested, check the terms before using them."],"risks":["robots.txt forbids some AI crawlers: GPTBot, Google-Extended."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":43,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","Google-Extended"],"ai_crawlers_blocked_count":2,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":["WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["login.alibaba.com/getEnvironment.do","s.alicdn.com/@xconfig/EcoNet/networkSpeedMap","s.alicdn.com/@xconfig/header/renderHeaderVersion","insights.alibaba.com/openservice/gatewayService","s.alicdn.com/@xconfig/header/subHeaderConfig","s.alicdn.com/@xconfig/header/deferAssetsConfig"],"inner_page":null},"performance":{"response_ms":663,"bytes":221554,"redirects":0,"cookies_set":5},"why_this_score":[{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 2 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/alibaba.com.json"},{"schema":"ratradar.site/v1","host":"www.taobao.com","name":"Taobao","category":"shop","country":"China","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","A browser load called JSON endpoints that look like data, for example umdcv4.taobao.com/repTw.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"Tengine"},"robots":{"found":true,"disallow_all":false,"disallow_rules":1,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["umdcv4.taobao.com/repTw.json","umdc.taobao.com/wcfg.json","hudong.alicdn.com/api/online/data/v3/c45746794e4a4de99af2b9fb4f65e0d2.js","h5api.m.taobao.com/h5/mtop.tmall.kangaroo.core.service.route.aldlampservicefixedresv2/1.0/","alidt.alicdn.com/alilog/configs/sdk/common.json","h5api.m.taobao.com/h5/mtop.user.getusersimple/1.0/"],"inner_page":null},"performance":{"response_ms":324,"bytes":93201,"redirects":0,"cookies_set":0},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/taobao.com.json"},{"schema":"ratradar.site/v1","host":"www.rakuten.co.jp","name":"Rakuten","category":"shop","country":"Japan","verdict":"open","hostility_score":5,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Read the JSON-LD on the page (Corporation, WebSite). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example cdn.rmc.contents.rakuten.co.jp/rmccx/d40535ac09aac0ef32f8a23b8bb8e04bc2e8f9d6b0981fa98a16d0382f558d83/blocks.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"Apache"},"robots":{"found":true,"disallow_all":false,"disallow_rules":12,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":["Corporation","WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["cdn.rmc.contents.rakuten.co.jp/rmccx/d40535ac09aac0ef32f8a23b8bb8e04bc2e8f9d6b0981fa98a16d0382f558d83/blocks.json","rmc-cx.api.rakuten.co.jp/v3/web/blocks","rmc-cx.api.rakuten.co.jp/v3/web/events","ups.im-apps.net/1000128/uid"],"inner_page":null},"performance":{"response_ms":5113,"bytes":381486,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Slow response (5113 ms)","points":5}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/rakuten.co.jp.json"},{"schema":"ratradar.site/v1","host":"www.naver.com","name":"Naver","category":"search","country":"South Korea","verdict":"hostile","hostility_score":50,"recommended_approach":"respect_robots_skip","steps":["robots.txt disallows every crawler. Do not scrape it. Look for an official API, a data license, or a partner feed."],"risks":["Scraping against an explicit robots.txt ban is the quickest way to a legal or blocking problem."],"can_scrape":{"plain_http":false,"headless_browser":true,"needs_proxies":false,"official_api_advised":true},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"nfront"},"robots":{"found":true,"disallow_all":true,"disallow_rules":1,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":271,"bytes":289814,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15},{"label":"robots.txt disallows everything","points":10},{"label":"Raised to at least 50 because robots.txt disallows everything","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt bans every crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/naver.com.json"},{"schema":"ratradar.site/v1","host":"www.flipkart.com","name":"Flipkart","category":"shop","country":"India","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","A browser load called JSON endpoints that look like data, for example 1.rome.api.flipkart.com/4/user/state. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":65,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.flipkart.com/sitemap_v_view-browse.xml.gz","https://www.flipkart.com/sitemap_s_store-browse.xml.gz","https://www.flipkart.com/sitemap_q_query-browse.xml.gz"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["1.rome.api.flipkart.com/4/user/state"],"inner_page":null},"performance":{"response_ms":867,"bytes":400000,"redirects":0,"cookies_set":6},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/flipkart.com.json"},{"schema":"ratradar.site/v1","host":"www.zomato.com","name":"Zomato","category":"food","country":"India","verdict":"guarded","hostility_score":45,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":15035,"bytes":null,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"No answer to a plain request","points":40},{"label":"Slow response (15035 ms)","points":5}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":0,"headless_browser_status":0,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/zomato.com.json"},{"schema":"ratradar.site/v1","host":"www.mercadolibre.com","name":"Mercado Libre","category":"market","country":"Argentina","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls."],"risks":["Hostile to automation. Keep volumes low and cache.","robots.txt forbids 5 of 14 known AI crawlers (GPTBot, ChatGPT-User, ClaudeBot, PerplexityBot, ...). An agent that identifies as an AI crawler should not fetch this site.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":5,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","ClaudeBot","PerplexityBot","Amazonbot"],"ai_crawlers_blocked_count":5,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":270,"bytes":18087,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 5 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/mercadolibre.com.json"},{"schema":"ratradar.site/v1","host":"www.globo.com","name":"Globo","category":"news","country":"Brazil","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: http://www.globo.com/sitemap-image.xml","A browser load called JSON endpoints that look like data, for example mab.g.globo/choose. Untested, check the terms before using them."],"risks":["robots.txt forbids 3 of 14 known AI crawlers (GPTBot, CCBot, Google-Extended). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":10,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","CCBot","Google-Extended"],"ai_crawlers_blocked_count":3,"sitemap_urls":["http://www.globo.com/sitemap-image.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["mab.g.globo/choose"],"inner_page":null},"performance":{"response_ms":1907,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 3 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/globo.com.json"},{"schema":"ratradar.site/v1","host":"www.takealot.com","name":"Takealot","category":"shop","country":"South Africa","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":53,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.takealot.com/sitemap.xml"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":35,"bytes":5599,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/takealot.com.json"},{"schema":"ratradar.site/v1","host":"www.jumia.com.ng","name":"Jumia","category":"shop","country":"Nigeria","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans.","robots.txt forbids some AI crawlers: CCBot."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":165,"crawl_delay_seconds":null,"ai_crawlers_blocked":["CCBot"],"ai_crawlers_blocked_count":1,"sitemap_urls":["https://static.jumia.com.ng/index-sitemap.xml","https://www.jumia.com.ng/slp/nc/sitemap/sitemap.xml"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":29,"bytes":5603,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"strong","signals":[{"kind":"robots_txt","label":"robots.txt blocks 1 AI crawler"},{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/jumia.com.ng.json"},{"schema":"ratradar.site/v1","host":"www.canva.com","name":"Canva","category":"saas","country":"Australia","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls.","Enumerate pages from the sitemap: https://sitemap.canva.com/sitemap_index.xml"],"risks":["Hostile to automation. Keep volumes low and cache.","robots.txt forbids 8 of 14 known AI crawlers (GPTBot, ClaudeBot, Claude-Web, anthropic-ai, ...). An agent that identifies as an AI crawler should not fetch this site.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":106,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ClaudeBot","Claude-Web","anthropic-ai","CCBot","Bytespider","Applebot-Extended","Meta-ExternalAgent"],"ai_crawlers_blocked_count":8,"sitemap_urls":["https://sitemap.canva.com/sitemap_index.xml","https://www.canva.com/landing_page_sitemap_index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.canva.com/ar/help/courses/","status":null,"json_ld_types":[],"skipped":"robots.txt disallows this path","sitemap":"https://sitemap.canva.com/sitemap_0.xml"}},"performance":{"response_ms":120,"bytes":9315,"redirects":0,"cookies_set":4},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15},{"label":"Cloudflare detected","points":6},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 8 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/canva.com.json"},{"schema":"ratradar.site/v1","host":"www.seek.com.au","name":"Seek","category":"jobs","country":"Australia","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":6,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":61,"bytes":5615,"redirects":1,"cookies_set":0},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/seek.com.au.json"},{"schema":"ratradar.site/v1","host":"www.bestbuy.com","name":"Best Buy","category":"shop","country":"United States","verdict":"open","hostility_score":15,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://sitemaps.bestbuy.com/sitemaps_discover_learn.xml"],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":182,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://sitemaps.bestbuy.com/sitemaps_discover_learn.xml","https://sitemaps.bestbuy.com/sitemaps_pdp.xml","https://sitemaps.bestbuy.com/sitemaps_promos.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.bestbuy.com/site/3-year-accidental-geek-squad-protection/6003001.p?skuId=6003001","status":0,"json_ld_types":[],"skipped":null,"sitemap":"https://sitemaps.bestbuy.com/sitemaps_pdp.0000.xml.gz"}},"performance":{"response_ms":8165,"bytes":400000,"redirects":0,"cookies_set":10},"why_this_score":[{"label":"Akamai detected","points":10},{"label":"Slow response (8165 ms)","points":5}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":0,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/bestbuy.com.json"},{"schema":"ratradar.site/v1","host":"www.homedepot.com","name":"Home Depot","category":"shop","country":"United States","verdict":"guarded","hostility_score":30,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429.","Enumerate pages from the sitemap: https://www.homedepot.com/sitemap/main.xml","A browser load called JSON endpoints that look like data, for example swasc.homedepot.com/ee/or2/v1/identity/acquire. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai","PerimeterX"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":45,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.homedepot.com/sitemap/main.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["swasc.homedepot.com/ee/or2/v1/identity/acquire","gcp-runtime.getamigo.io/v2/639e1c8d99ca8d63ed7c2173faca59ea10d454ca14d8d6c34e3fd9353b9fa410/plugins","swasc.homedepot.com/ee/or2/v1/interact","www.homedepot.com/yFPi4U/Kts6ec/dQSMApd/8E96AAhE/7bO5p8kzcpQfrk/DmZ0VSZvWw/bz0/lcUoQDys","clickstream-killswitch-api.hd-personalization-prod.gcp.homedepot.com/clickstream-killswitch/v1/detail"],"inner_page":{"url":"https://www.homedepot.com/c/ap/a-new-roof-made-easy/9ba683603be9fa5395fab9017b777ba5","status":403,"json_ld_types":[],"skipped":null,"sitemap":"https://www.homedepot.com/sitemap/C/Content/Article/Article-0.xml"}},"performance":{"response_ms":369,"bytes":400000,"redirects":0,"cookies_set":10},"why_this_score":[{"label":"Akamai detected","points":10},{"label":"PerimeterX seen by the browser","points":20}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/homedepot.com.json"},{"schema":"ratradar.site/v1","host":"www.lowes.com","name":"Lowe's","category":"shop","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here."],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":""},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":64,"bytes":5071,"redirects":0,"cookies_set":14},"why_this_score":[{"label":"Akamai detected","points":10},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/lowes.com.json"},{"schema":"ratradar.site/v1","host":"www.costco.com","name":"Costco","category":"shop","country":"United States","verdict":"open","hostility_score":10,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.costco.com/sitemap_lw_index.xml"],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":54,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.costco.com/sitemap_lw_index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.costco.com/home-alarm-systems-automation.html?brand=asus&refine=||Brand_attr-ASUS","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.costco.com/sitemap_lw_r_000.xml"}},"performance":{"response_ms":206,"bytes":400000,"redirects":0,"cookies_set":3},"why_this_score":[{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/costco.com.json"},{"schema":"ratradar.site/v1","host":"www.newegg.com","name":"Newegg","category":"shop","country":"United States","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.newegg.com/xmlsitemap/2019_Updated_Confidential_siteindex_USA.xml","Read the JSON-LD on the page (WebSite, Organization). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.newegg.com/api/MegaNavPanels. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":70,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.newegg.com/xmlsitemap/2019_Updated_Confidential_siteindex_USA.xml","https://www.newegg.com/xmlsitemap/2019_Updated_Confidential_siteindex_USA_Seller.xml","https://www.newegg.com/xmlsitemap/ProductListKeywords_USA.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["WebSite","Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.newegg.com/api/MegaNavPanels","www.newegg.com/api/CountryApi","www.newegg.com/api/HomeMainSaleStore","www.newegg.com/ailab/api/init","www.newegg.com/home/api/RecommendedModules","secure.newegg.com/api/GetMessageInfos"],"inner_page":{"url":"https://www.newegg.com/p/pl?d=%2410","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.newegg.com/xmlsitemap/ProductListKeywords_USA.xml"}},"performance":{"response_ms":67,"bytes":400000,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/newegg.com.json"},{"schema":"ratradar.site/v1","host":"www.wayfair.com","name":"Wayfair","category":"shop","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Enumerate pages from the sitemap: https://www.wayfair.com/seo-category-index.xml"],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare","DataDome","PerimeterX"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":51,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.wayfair.com/seo-category-index.xml","https://www.wayfair.com/seo-keyword-index.xml","https://www.wayfair.com/seo-sb0-index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.wayfair.com/decor-pillows/pdp/red-barrel-studio-potted-basil-4-piece-wrapped-canvas-painting-set-w006238502.html","status":429,"json_ld_types":[],"skipped":null,"sitemap":"https://www.wayfair.com/seo-pdp-sitemap~0.xml"}},"performance":{"response_ms":367,"bytes":400000,"redirects":0,"cookies_set":3},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"DataDome detected","points":20},{"label":"PerimeterX detected","points":20},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":429,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/wayfair.com.json"},{"schema":"ratradar.site/v1","host":"www.ikea.com","name":"IKEA","category":"shop","country":"Netherlands","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.ikea.com/sitemaps/sitemap.xml","Read the JSON-LD on the page (Organization). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.ikea.com/ext/mylocation/iso. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":92,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.ikea.com/sitemaps/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.ikea.com/ext/mylocation/iso"],"inner_page":{"url":"https://www.ikea.com/ee/et/p/zebrasaev-laelamp-valge-plast-10522481/","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.ikea.com/sitemaps/prod-et-EE_1.xml"}},"performance":{"response_ms":270,"bytes":400000,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/ikea.com.json"},{"schema":"ratradar.site/v1","host":"www.nike.com","name":"Nike","category":"shop","country":"United States","verdict":"open","hostility_score":10,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.nike.com/sitemap-us-help.xml","Read the JSON-LD on the page (WebPage, Organization, WebSite). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.nike.com/static/dotcom/shared-config/public/us/config.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"unified-edge-router"},"robots":{"found":true,"disallow_all":false,"disallow_rules":18,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.nike.com/sitemap-us-help.xml","https://www.nike.com/sitemap-v2-landingpage-index.xml","https://www.nike.com/sitemap-v2-pdp-index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebPage","Organization","WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.nike.com/static/dotcom/shared-config/public/us/config.json","api.nike.com/search/suggestions/v1","www.nike.com/fragments/bellotti-playcard","www.nike.com/fragments/async-chat","www.nike.com/fragments/error-modal"],"inner_page":{"url":"https://www.nike.com/at/en/t/24-7-impossiblysoft-mens-cardigan-klvlqAn7","status":200,"json_ld_types":["ProductGroup"],"skipped":null,"sitemap":"https://www.nike.com/sitemap-v2-pdp-en-at.xml"}},"performance":{"response_ms":356,"bytes":400000,"redirects":0,"cookies_set":13},"why_this_score":[{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/nike.com.json"},{"schema":"ratradar.site/v1","host":"www.adidas.com","name":"Adidas","category":"shop","country":"Germany","verdict":"blocked","hostility_score":80,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"AkamaiNetStorage"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":59,"bytes":3367,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/adidas.com.json"},{"schema":"ratradar.site/v1","host":"www.zara.com","name":"Zara","category":"shop","country":"Spain","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Akamai"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":53,"bytes":366,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/zara.com.json"},{"schema":"ratradar.site/v1","host":"www2.hm.com","name":"H&M","category":"shop","country":"Sweden","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Akamai"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"AkamaiGHost"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":46,"bytes":365,"redirects":0,"cookies_set":3},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/www2.hm.com.json"},{"schema":"ratradar.site/v1","host":"www.shein.com","name":"Shein","category":"shop","country":"China","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Read the JSON-LD on the page (OnlineStore). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example us.shein.com/bff-api/user-api/privacy-banner-api/query_user_latest_gpc. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":7,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.shein.com/sitemap-index.xml"]},"machine_readable":{"sitemap":false,"llms_txt":true,"json_ld_types":["OnlineStore"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["us.shein.com/bff-api/user-api/privacy-banner-api/query_user_latest_gpc","us.shein.com/bff-api/user-api/privacy-banner-api/intercept_list","us.shein.com/bff-api/user-api/privacy-banner-api/get_private_banner_gpc","us.shein.com/bff-api/user-api/privacy-banner-api/save_user_gpc","us.shein.com/bff-api/user-api/privacy-banner-api/get_private_banner","us.shein.com/bff-api/user-api/privacy-banner-api/query_user_latest_cookie"],"inner_page":null},"performance":{"response_ms":960,"bytes":400000,"redirects":1,"cookies_set":7},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/shein.com.json"},{"schema":"ratradar.site/v1","host":"www.temu.com","name":"Temu","category":"shop","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Read the JSON-LD on the page (Organization, WebSite, FAQPage). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.temu.com/api/server/_stm. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"Server"},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.temu.com/sitemap-index.xml"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":["Organization","WebSite","FAQPage"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.temu.com/api/server/_stm","www.temu.com/api/adx/cm/ttc","www.temu.com/api/static/config/get_front_end_page_config","www.temu.com/api/jade/neutrino/dnld/qr/get","www.temu.com/api/adx/cm/match-tag","www.temu.com/api/poppy/v2/search_activation"],"inner_page":null},"performance":{"response_ms":729,"bytes":400000,"redirects":0,"cookies_set":4},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/temu.com.json"},{"schema":"ratradar.site/v1","host":"www.aliexpress.com","name":"AliExpress","category":"shop","country":"China","verdict":"open","hostility_score":5,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.aliexpress.com/sitemap/en-https-item-item-2026-P0-sitemap-index.xml","A browser load called JSON endpoints that look like data, for example o.alicdn.com/ae-ug-fe/ae-ug-plugin-config/cookie-config.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"Tengine"},"robots":{"found":true,"disallow_all":false,"disallow_rules":34,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.aliexpress.com/sitemap/en-https-item-item-2026-P0-sitemap-index.xml","https://www.aliexpress.com/sitemap/en-https-item-item-2026-P1-sitemap-index.xml","https://www.aliexpress.com/sitemap/en-https-wholesale-2025-w-commercial-local-sitemap-index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["o.alicdn.com/ae-ug-fe/ae-ug-plugin-config/cookie-config.json","o.alicdn.com/gcom/ug-member-config/config.json","recom-acs.aliexpress.us/h5/mtop.relationrecommend.aliexpressrecommend.recommend/1.0/","us.creativecdn.com/tags/v2"],"inner_page":{"url":"https://www.aliexpress.com/item/10000000214274.html","status":302,"json_ld_types":[],"skipped":null,"sitemap":"https://www.aliexpress.com/sitemap/en-https-item-item-2026-P0-0.xml.gz"}},"performance":{"response_ms":1760,"bytes":null,"redirects":13,"cookies_set":6},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":302,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/aliexpress.com.json"},{"schema":"ratradar.site/v1","host":"www.zalando.com","name":"Zalando","category":"shop","country":"Germany","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"AmazonS3"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":273,"bytes":48385,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/zalando.com.json"},{"schema":"ratradar.site/v1","host":"www.otto.de","name":"Otto","category":"shop","country":"Germany","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.otto.de/category-sitemap/sitemap_index.xml.gz","Read the JSON-LD on the page (Organization). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.otto.de/san-squirrel-suggest-api/completion. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"Edge Proxy"},"robots":{"found":true,"disallow_all":false,"disallow_rules":4,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.otto.de/category-sitemap/sitemap_index.xml.gz","https://www.otto.de/products-sitemap/sitemap_einrichten-wohnen_index.xml","https://www.otto.de/products-sitemap/sitemap_fashion-lifestyle_index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.otto.de/san-squirrel-suggest-api/completion","www.otto.de/order/basket/amount.json","www.otto.de/reco-core/cinemas/lastseensearch","www.otto.de/promo-explicit-api/api/brands/(und.(ist.marke.home-affaire).(~.(v.1)))","www.otto.de/wishlist-view-amount-json/","www.otto.de/nav-customizer/local-recos"],"inner_page":{"url":"https://www.otto.de/p/ace-alkoholtestgeraet-der-alkoholtester-mit-austauschbarem-sensor-fuer-dauerhafte-107045-auswechselbarer-sensor-inkl-display-CS053Y0LN/","status":400,"json_ld_types":[],"skipped":null,"sitemap":"https://www.otto.de/products-sitemap/sitemap_sonstige_1.xml.gz"}},"performance":{"response_ms":1365,"bytes":327556,"redirects":0,"cookies_set":3},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/otto.de.json"},{"schema":"ratradar.site/v1","host":"allegro.pl","name":"Allegro","category":"market","country":"Poland","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["DataDome"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":33,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":true,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":539,"bytes":780,"redirects":0,"cookies_set":3},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"DataDome detected","points":20}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":0,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/allegro.pl.json"},{"schema":"ratradar.site/v1","host":"www.cdiscount.com","name":"Cdiscount","category":"shop","country":"France","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls."],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":244,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":182,"bytes":13993,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15},{"label":"Cloudflare detected","points":6},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/cdiscount.com.json"},{"schema":"ratradar.site/v1","host":"www.mercari.com","name":"Mercari","category":"market","country":"Japan","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies.","Enumerate pages from the sitemap: https://www.mercari.com/us-sitemap-index.xml"],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":35,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.mercari.com/us-sitemap-index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":33,"bytes":5555,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/mercari.com.json"},{"schema":"ratradar.site/v1","host":"poshmark.com","name":"Poshmark","category":"market","country":"United States","verdict":"open","hostility_score":8,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Read the JSON-LD on the page (WebSite). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example api.tx4.pw.adn.cloud/0GL2UE/974240149215149012. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront","Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"Version 1.0"},"robots":{"found":true,"disallow_all":false,"disallow_rules":42,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":["WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["api.tx4.pw.adn.cloud/0GL2UE/974240149215149012","ipv4.pdscrb.com/","ct.pinterest.com/user/"],"inner_page":null},"performance":{"response_ms":257,"bytes":400000,"redirects":0,"cookies_set":4},"why_this_score":[{"label":"CloudFront detected","points":2},{"label":"Cloudflare seen by the browser","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/poshmark.com.json"},{"schema":"ratradar.site/v1","host":"www.depop.com","name":"Depop","category":"market","country":"United Kingdom","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":84,"bytes":256638,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/depop.com.json"},{"schema":"ratradar.site/v1","host":"www.vinted.com","name":"Vinted","category":"market","country":"Ireland","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://vinted.com/sitemap-us.xml","Read the JSON-LD on the page (WebPage). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.vinted.com/relay/events. Untested, check the terms before using them."],"risks":["robots.txt forbids 5 of 14 known AI crawlers (GPTBot, ClaudeBot, CCBot, Google-Extended, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":6,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ClaudeBot","CCBot","Google-Extended","Amazonbot"],"ai_crawlers_blocked_count":5,"sitemap_urls":["https://vinted.com/sitemap-us.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebPage"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.vinted.com/relay/events","www.vinted.com/api/v2/banners","www.vinted.com/api/v2/info_banners/news_feed","api.www.vinted.com/j3r4zw/v1/config"],"inner_page":{"url":"https://www.vinted.com/items/8880526957-kids-polo-dress","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.vinted.com/sitemaps/us/items.xml"}},"performance":{"response_ms":970,"bytes":400000,"redirects":0,"cookies_set":9},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 5 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/vinted.com.json"},{"schema":"ratradar.site/v1","host":"www.craigslist.org","name":"Craigslist","category":"market","country":"United States","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.craigslist.org/sitemap-index-categories-and-hubs.xml","Read the JSON-LD on the page (WebSite). It is cleaner than the markup."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":7,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.craigslist.org/sitemap-index-categories-and-hubs.xml","https://www.craigslist.org/sitemap-index-postings-00.xml","https://www.craigslist.org/sitemap-index-postings-01.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.craigslist.org/view/d/dublin-best-movers-truck-moving-pros-no/gJTU5HTSLhHGjEBG7zEejJ","status":200,"json_ld_types":["BreadcrumbList"],"skipped":null,"sitemap":"https://www.craigslist.org/sitemap-postings-2026-10-11-sfo-bbb.xml"}},"performance":{"response_ms":446,"bytes":61168,"redirects":2,"cookies_set":4},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/craigslist.org.json"},{"schema":"ratradar.site/v1","host":"offerup.com","name":"OfferUp","category":"market","country":"United States","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Read the JSON-LD on the page (WebSite). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example aeg.offerup-apis.com/adevent. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":48,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://c.offerup.com/item-detail-index.xml.gz","https://c.offerup.com/explore-regional-topics-index.xml.gz","https://c.offerup.com/explore-global-topics-index.xml.gz"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":["WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["aeg.offerup-apis.com/adevent","firebase.googleapis.com/v1alpha/projects/-/apps/1:908780932154:web:f8c45204ecb8baa2b3f903/webConfig","geo.privacymanager.io/","prodregistryv2.org/v1/rgstr"],"inner_page":null},"performance":{"response_ms":574,"bytes":400000,"redirects":0,"cookies_set":10},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/offerup.com.json"},{"schema":"ratradar.site/v1","host":"www.gumtree.com","name":"Gumtree","category":"market","country":"United Kingdom","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.gumtree.com/info/cars/sitemap.xml","Read the JSON-LD on the page (Organization, WebSite, WebPage). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example nodals.io/p/v1/d990766f/config. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":140,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.gumtree.com/info/cars/sitemap.xml","https://www.gumtree.com/xmlsitemap/vip/sitemap.xml","https://www.gumtree.com/xmlsitemap/location/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["Organization","WebSite","WebPage"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["nodals.io/p/v1/d990766f/config"],"inner_page":{"url":"https://www.gumtree.com/news/cars/used-ford-puma-gen-e-for-under-18000","status":200,"json_ld_types":["NewsArticle","BreadcrumbList"],"skipped":null,"sitemap":"https://www.gumtree.com/news/news-sitemap-1.xml"}},"performance":{"response_ms":883,"bytes":400000,"redirects":0,"cookies_set":10},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/gumtree.com.json"},{"schema":"ratradar.site/v1","host":"www.olx.in","name":"OLX","category":"market","country":"India","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans.","robots.txt forbids 3 of 14 known AI crawlers (CCBot, Bytespider, Amazonbot). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Akamai"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"AkamaiGHost"},"robots":{"found":true,"disallow_all":false,"disallow_rules":23,"crawl_delay_seconds":null,"ai_crawlers_blocked":["CCBot","Bytespider","Amazonbot"],"ai_crawlers_blocked_count":3,"sitemap_urls":["https://www.olx.in/sitemapindex.xml","https://www.olx.in/blog/sitemap_index.xml"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":50,"bytes":364,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"strong","signals":[{"kind":"robots_txt","label":"robots.txt blocks 3 AI crawlers"},{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/olx.in.json"},{"schema":"ratradar.site/v1","host":"www.kleinanzeigen.de","name":"Kleinanzeigen","category":"market","country":"Germany","verdict":"open","hostility_score":10,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.kleinanzeigen.de/sitemap_index.xml","Read the JSON-LD on the page (Organization, WebSite, WebPage). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.kleinanzeigen.de/khuTU10CL/ZH/7WKW6Ng/atG7QbJrV62zmVGiYm/SGNYRA/fjQcNEAb/GQU. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":251,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.kleinanzeigen.de/sitemap_index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["Organization","WebSite","WebPage"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.kleinanzeigen.de/khuTU10CL/ZH/7WKW6Ng/atG7QbJrV62zmVGiYm/SGNYRA/fjQcNEAb/GQU","www.kleinanzeigen.de/_actions/redWeb.gallery.getGallery/"],"inner_page":{"url":"https://www.kleinanzeigen.de/s-seehausen-/l13490","status":200,"json_ld_types":["WebSite","ImageObject"],"skipped":null,"sitemap":"https://www.kleinanzeigen.de/sitemap_cities.xml"}},"performance":{"response_ms":332,"bytes":290779,"redirects":0,"cookies_set":4},"why_this_score":[{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/kleinanzeigen.de.json"},{"schema":"ratradar.site/v1","host":"www.leboncoin.fr","name":"Leboncoin","category":"market","country":"France","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans.","robots.txt forbids some AI crawlers: Bytespider."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["DataDome","CloudFront"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":["Bytespider"],"ai_crawlers_blocked_count":1,"sitemap_urls":["https://www.leboncoin.fr/auto-main.xml"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":505,"bytes":771,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"DataDome detected","points":20},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"strong","signals":[{"kind":"robots_txt","label":"robots.txt blocks 1 AI crawler"},{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/leboncoin.fr.json"},{"schema":"ratradar.site/v1","host":"www.coupang.com","name":"Coupang","category":"shop","country":"South Korea","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Akamai"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"AkamaiGHost"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":48,"bytes":369,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/coupang.com.json"},{"schema":"ratradar.site/v1","host":"shopee.sg","name":"Shopee","category":"shop","country":"Singapore","verdict":"open","hostility_score":15,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls.","robots.txt asks for a crawl delay of 1 seconds.","A browser load called JSON endpoints that look like data, for example deo.shopeemobile.com/shopee/stm-sg-live/shopee-pcmall-live-sg/en.col159.1755192911.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"SGW"},"robots":{"found":true,"disallow_all":false,"disallow_rules":18,"crawl_delay_seconds":1,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["deo.shopeemobile.com/shopee/stm-sg-live/shopee-pcmall-live-sg/en.col159.1755192911.json","deo.shopeemobile.com/shopee/stm-sg-live/shopee-pcmall-live-sg/en.col722.1789355076.json","deo.shopeemobile.com/shopee/stm-sg-live/shopee-pcmall-live-sg/en.col134.1759225570.json","deo.shopeemobile.com/shopee/stm-sg-live/shopee-pcmall-live-sg/en.col619.1780454458.json","deo.shopeemobile.com/shopee/stm-sg-live/shopee-pcmall-live-sg/en.col179.1781172837.json","deo.shopeemobile.com/shopee/stm-sg-live/shopee-pcmall-live-sg/en.col60.1755192897.json"],"inner_page":null},"performance":{"response_ms":1270,"bytes":163435,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/shopee.sg.json"},{"schema":"ratradar.site/v1","host":"www.lazada.com","name":"Lazada","category":"shop","country":"Singapore","verdict":"open","hostility_score":11,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"Tengine"},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":5658,"bytes":27331,"redirects":1,"cookies_set":2},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"Slow response (5658 ms)","points":5}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/lazada.com.json"},{"schema":"ratradar.site/v1","host":"www.tokopedia.com","name":"Tokopedia","category":"shop","country":"Indonesia","verdict":"guarded","hostility_score":45,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":110,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.tokopedia.com/sitemap/category-index.xml","https://www.tokopedia.com/sitemap/deals-index.xml","https://www.tokopedia.com/sitemap/events-index.xml"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":15040,"bytes":null,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"No answer to a plain request","points":40},{"label":"Slow response (15040 ms)","points":5}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":0,"headless_browser_status":0,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/tokopedia.com.json"},{"schema":"ratradar.site/v1","host":"www.rappi.com","name":"Rappi","category":"food","country":"Colombia","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Read the JSON-LD on the page (Organization, ItemList, WebSite). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.rappi.com.mx/api/cloudfront-location. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"nginx/1.27.5"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":["Organization","ItemList","WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.rappi.com.mx/api/cloudfront-location","www.rappi.com.mx/mf-header/rappi-microfront-manifest.json","services.mxgrability.rappi.com/api/rocket/v2/guest/passport/","services.mxgrability.rappi.com/api/rocket/v2/guest","r.lr-ingest.com/i"],"inner_page":null},"performance":{"response_ms":261,"bytes":171904,"redirects":1,"cookies_set":2},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/rappi.com.json"},{"schema":"ratradar.site/v1","host":"www.noon.com","name":"Noon","category":"shop","country":"United Arab Emirates","verdict":"guarded","hostility_score":45,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":15040,"bytes":null,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"No answer to a plain request","points":40},{"label":"Slow response (15040 ms)","points":5}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":0,"headless_browser_status":0,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/noon.com.json"},{"schema":"ratradar.site/v1","host":"www.kaufland.de","name":"Kaufland","category":"shop","country":"Germany","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":47,"bytes":10948,"redirects":0,"cookies_set":4},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/kaufland.de.json"},{"schema":"ratradar.site/v1","host":"www.argos.co.uk","name":"Argos","category":"shop","country":"United Kingdom","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Akamai"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":66,"bytes":373,"redirects":0,"cookies_set":7},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/argos.co.uk.json"},{"schema":"ratradar.site/v1","host":"www.tesco.com","name":"Tesco","category":"shop","country":"United Kingdom","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Akamai"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"Tesco Web Server"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":49,"bytes":365,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/tesco.com.json"},{"schema":"ratradar.site/v1","host":"www.carrefour.fr","name":"Carrefour","category":"shop","country":"France","verdict":"blocked","hostility_score":76,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":5,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":118,"bytes":400000,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/carrefour.fr.json"},{"schema":"ratradar.site/v1","host":"www.lidl.de","name":"Lidl","category":"shop","country":"Germany","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.lidl.de/static/sitemap.xml","Read the JSON-LD on the page (Organization). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.lidl.de/n/api/asset-dts. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"myracloud"},"robots":{"found":true,"disallow_all":false,"disallow_rules":12,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.lidl.de/static/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.lidl.de/n/api/asset-dts","www.lidl.de/s/translation/de-DE.json"],"inner_page":{"url":"https://www.lidl.de/p/edition-wuerttemberg-schwarzriesling-qba-lieblich-literflasche-rotwein-2023/p100000271","status":200,"json_ld_types":["Product","Organization"],"skipped":null,"sitemap":"https://www.lidl.de/p/export/DE/de/product_sitemap.xml.gz"}},"performance":{"response_ms":1010,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/lidl.de.json"},{"schema":"ratradar.site/v1","host":"www.aldi.us","name":"Aldi","category":"shop","country":"United States","verdict":"hostile","hostility_score":50,"recommended_approach":"respect_robots_skip","steps":["robots.txt disallows every crawler. Do not scrape it. Look for an official API, a data license, or a partner feed.","Enumerate pages from the sitemap: https://www.aldi.us/sitemaps/storefront_pro/www_aldi_us/sitemap.xml","A browser load called JSON endpoints that look like data, for example cdn0.forter.com/e44177b6b805/292c7b4fc07248fdb0a9613d90cf92cc/prop.json. Untested, check the terms before using them."],"risks":["Scraping against an explicit robots.txt ban is the quickest way to a legal or blocking problem.","robots.txt forbids some AI crawlers: PerplexityBot, Meta-ExternalAgent."],"can_scrape":{"plain_http":false,"headless_browser":true,"needs_proxies":false,"official_api_advised":true},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"nginx"},"robots":{"found":true,"disallow_all":true,"disallow_rules":1,"crawl_delay_seconds":null,"ai_crawlers_blocked":["PerplexityBot","Meta-ExternalAgent"],"ai_crawlers_blocked_count":2,"sitemap_urls":["https://www.aldi.us/sitemaps/storefront_pro/www_aldi_us/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["cdn0.forter.com/e44177b6b805/292c7b4fc07248fdb0a9613d90cf92cc/prop.json","cdn0.forter.com/e44177b6b805/292c7b4fc07248fdb0a9613d90cf92cc/wpt.json","www.aldi.us/ahoy/visits","www.aldi.us/graphql"],"inner_page":null},"performance":{"response_ms":3733,"bytes":400000,"redirects":1,"cookies_set":8},"why_this_score":[{"label":"CloudFront detected","points":2},{"label":"robots.txt disallows everything","points":10},{"label":"Slow response (3733 ms)","points":5},{"label":"Raised to at least 50 because robots.txt disallows everything","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt bans every crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/aldi.us.json"},{"schema":"ratradar.site/v1","host":"www.instacart.com","name":"Instacart","category":"food","country":"United States","verdict":"hostile","hostility_score":50,"recommended_approach":"respect_robots_skip","steps":["robots.txt disallows every crawler. Do not scrape it. Look for an official API, a data license, or a partner feed.","Read the JSON-LD on the page (FAQPage, WebSite). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.instacart.com/graphql. Untested, check the terms before using them."],"risks":["Scraping against an explicit robots.txt ban is the quickest way to a legal or blocking problem.","robots.txt forbids some AI crawlers: PerplexityBot, Meta-ExternalAgent."],"can_scrape":{"plain_http":false,"headless_browser":true,"needs_proxies":false,"official_api_advised":true},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"nginx"},"robots":{"found":true,"disallow_all":true,"disallow_rules":1,"crawl_delay_seconds":null,"ai_crawlers_blocked":["PerplexityBot","Meta-ExternalAgent"],"ai_crawlers_blocked_count":2,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":["FAQPage","WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.instacart.com/graphql","www.instacart.com/ahoy/visits"],"inner_page":null},"performance":{"response_ms":1963,"bytes":400000,"redirects":0,"cookies_set":7},"why_this_score":[{"label":"CloudFront detected","points":2},{"label":"robots.txt disallows everything","points":10},{"label":"Raised to at least 50 because robots.txt disallows everything","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt bans every crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/instacart.com.json"},{"schema":"ratradar.site/v1","host":"www.chewy.com","name":"Chewy","category":"shop","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies.","Enumerate pages from the sitemap: https://www.chewy.com/app/sitemap/behijkmoqrrtttvvy-sitemap_index.xml"],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans.","robots.txt forbids some AI crawlers: Amazonbot."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Akamai","Kasada"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":44,"crawl_delay_seconds":null,"ai_crawlers_blocked":["Amazonbot"],"ai_crawlers_blocked_count":1,"sitemap_urls":["https://www.chewy.com/app/sitemap/behijkmoqrrtttvvy-sitemap_index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":173,"bytes":661,"redirects":0,"cookies_set":6},"why_this_score":[{"label":"HTTP 429 on a plain request","points":70},{"label":"Akamai detected","points":10},{"label":"Kasada detected","points":20}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 1 AI crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":429,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/chewy.com.json"},{"schema":"ratradar.site/v1","host":"www.sephora.com","name":"Sephora","category":"shop","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Akamai"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"AkamaiGHost"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":59,"bytes":367,"redirects":0,"cookies_set":16},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/sephora.com.json"},{"schema":"ratradar.site/v1","host":"www.ulta.com","name":"Ulta","category":"shop","country":"United States","verdict":"open","hostility_score":10,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.ulta.com/sitemap/index.xml","A browser load called JSON endpoints that look like data, for example www.ulta.com/dxl/graphql. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.ulta.com/sitemap/index.xml","https://www.ulta.com/stores/sitemap.xml","https://www.ulta.com/l/brand_filter_sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.ulta.com/dxl/graphql"],"inner_page":{"url":"https://www.ulta.com/shop/fragrance/womens-fragrance/perfume","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.ulta.com/sitemap/shop.xml"}},"performance":{"response_ms":161,"bytes":400000,"redirects":0,"cookies_set":6},"why_this_score":[{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/ulta.com.json"},{"schema":"ratradar.site/v1","host":"www.macys.com","name":"Macy's","category":"shop","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies.","A browser load called JSON endpoints that look like data, for example edge.fullstory.com/s/settings/104H4B/v1/web. Untested, check the terms before using them."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Akamai"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"AkamaiGHost"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["edge.fullstory.com/s/settings/104H4B/v1/web"],"inner_page":null},"performance":{"response_ms":57,"bytes":9900,"redirects":0,"cookies_set":6},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/macys.com.json"},{"schema":"ratradar.site/v1","host":"www.nordstrom.com","name":"Nordstrom","category":"shop","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.nordstrom.com/sitemap.xml"],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":45,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.nordstrom.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.nordstrom.com/s/kids-antora-triclimate-jacket-big-kid/7696410","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.nordstrom.com/product-sitemap-1.xml"}},"performance":{"response_ms":305,"bytes":254752,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/nordstrom.com.json"},{"schema":"ratradar.site/v1","host":"www.kohls.com","name":"Kohl's","category":"shop","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Akamai"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"AkamaiGHost"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":49,"bytes":367,"redirects":0,"cookies_set":32},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/kohls.com.json"},{"schema":"ratradar.site/v1","host":"www.dickssportinggoods.com","name":"Dick's","category":"shop","country":"United States","verdict":"blocked","hostility_score":90,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare","Akamai","Fastly","CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"AkamaiGHost"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":87,"bytes":1440,"redirects":0,"cookies_set":9},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"Cloudflare detected","points":6},{"label":"Akamai detected","points":10},{"label":"Fastly detected","points":2},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/dickssportinggoods.com.json"},{"schema":"ratradar.site/v1","host":"www.rei.com","name":"REI","category":"shop","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare","Akamai","CloudFront"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":47,"bytes":363,"redirects":0,"cookies_set":3},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6},{"label":"Akamai detected","points":10},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":0,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/rei.com.json"},{"schema":"ratradar.site/v1","host":"www.apple.com","name":"Apple Store","category":"shop","country":"United States","verdict":"open","hostility_score":10,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.apple.com/shop/sitemap.xml","Read the JSON-LD on the page (Organization, WebSite, WebPage). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.apple.com/api-www/global-elements/global-header/v1/flyouts. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"Apple"},"robots":{"found":true,"disallow_all":false,"disallow_rules":4,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.apple.com/shop/sitemap.xml","https://www.apple.com/autopush/sitemap/sitemap-index.xml","https://www.apple.com/newsroom/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["Organization","WebSite","WebPage"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.apple.com/api-www/global-elements/global-header/v1/flyouts","www.apple.com/search-services/suggestions/defaultlinks/","www.apple.com/us/shop/bag/status","www.apple.com/ac/localeswitcher/4/en_US/content/localeswitcher.json"],"inner_page":{"url":"https://www.apple.com/newsroom/2026/10/apple-the-kings-trust-launch-initiative-to-empower-young-creatives/","status":200,"json_ld_types":["Organization","NewsArticle"],"skipped":null,"sitemap":"https://www.apple.com/newsroom/sitemap.xml"}},"performance":{"response_ms":77,"bytes":253998,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/apple.com.json"},{"schema":"ratradar.site/v1","host":"www.samsung.com","name":"Samsung","category":"shop","country":"South Korea","verdict":"open","hostility_score":10,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.samsung.com/sitemap.xml","Read the JSON-LD on the page (WebSite, WebPage, Corporation). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.samsung.com/aemapi/v6/common/i18n/dict.en_US.b2c.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":49,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.samsung.com/sitemap.xml","https://www.samsung.com/africa_en/sitemap.xml","https://www.samsung.com/africa_fr/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["WebSite","WebPage","Corporation"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.samsung.com/aemapi/v6/common/i18n/dict.en_US.b2c.json","api-recommender.bigdata.samsung.com/prodrec/recommendation"],"inner_page":null},"performance":{"response_ms":154,"bytes":398152,"redirects":1,"cookies_set":5},"why_this_score":[{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/samsung.com.json"},{"schema":"ratradar.site/v1","host":"www.dell.com","name":"Dell","category":"shop","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Read the JSON-LD on the page (Organization, VideoObject). It is cleaner than the markup."],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":411,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.dell.com/index-sr-sitemap.xml.gz","https://www.dell.com/index-deals-href-sitemap.xml.gz","https://www.dell.com/index-product-pages-hreflang-sitemap.xml.gz"]},"machine_readable":{"sitemap":false,"llms_txt":true,"json_ld_types":["Organization","VideoObject"],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":289,"bytes":343811,"redirects":1,"cookies_set":8},"why_this_score":[{"label":"Akamai detected","points":10},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/dell.com.json"},{"schema":"ratradar.site/v1","host":"www.hp.com","name":"HP","category":"shop","country":"United States","verdict":"open","hostility_score":10,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","A browser load called JSON endpoints that look like data, for example www.hp.com/KJdyHbrjifBwNftUdWXeznVZ6QI/iV7QG8pzX6Nhcf/AXEJCm9FTgQ/TRUJfh/JdfnM. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"Apache"},"robots":{"found":true,"disallow_all":false,"disallow_rules":170,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.hp.com/sitemap-gb-en-mag.xml","https://www.hp.com/gb-en/shop/media/facets_url/products_url_gb_en.txt","https://www.hp.com/gb-en/shop/media/facets_url/facets_url_gb_en.txt"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.hp.com/KJdyHbrjifBwNftUdWXeznVZ6QI/iV7QG8pzX6Nhcf/AXEJCm9FTgQ/TRUJfh/JdfnM","global-navbar-backend.id.hp.com/api/v1/session/sso_check","experienceai.hp.com/api/auth","experienceai.hp.com/api/waitlist/context"],"inner_page":null},"performance":{"response_ms":221,"bytes":400000,"redirects":3,"cookies_set":4},"why_this_score":[{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/hp.com.json"},{"schema":"ratradar.site/v1","host":"www.lenovo.com","name":"Lenovo","category":"shop","country":"China","verdict":"hostile","hostility_score":60,"recommended_approach":"real_browser","steps":["Use a rendering browser, slow requests, and rotating proxies.","A browser load called JSON endpoints that look like data, for example www.lenovo.com/R1LiE/-k6S/CrVn/vBMP/N08/S5EOhDES9Euart/XUd3R1VALg/en1tKz/w6Jlo. Untested, check the terms before using them."],"risks":["Hostile to automation. Keep volumes low and cache."],"can_scrape":{"plain_http":false,"headless_browser":true,"needs_proxies":true,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"AkamaiGHost"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.lenovo.com/R1LiE/-k6S/CrVn/vBMP/N08/S5EOhDES9Euart/XUd3R1VALg/en1tKz/w6Jlo","openapi.lenovo.com/us/en/auth/assign_cookie","openapi.lenovo.com/us/en/api/sscmssite/fragment/getFragmentContentBatch","openapi.lenovo.com/us/en/ofp/search/suggest/selector"],"inner_page":null},"performance":{"response_ms":84,"bytes":366,"redirects":0,"cookies_set":3},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Akamai detected","points":10},{"label":"A real browser got in though a plain request did not","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/lenovo.com.json"},{"schema":"ratradar.site/v1","host":"store.steampowered.com","name":"Steam","category":"shop","country":"United States","verdict":"open","hostility_score":10,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","A browser load called JSON endpoints that look like data, for example store.steampowered.com/dynamicstore/saledata/. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":9,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["store.steampowered.com/dynamicstore/saledata/","store.steampowered.com/dynamicstore/userdata/"],"inner_page":null},"performance":{"response_ms":165,"bytes":400000,"redirects":0,"cookies_set":3},"why_this_score":[{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/store.steampowered.com.json"},{"schema":"ratradar.site/v1","host":"store.epicgames.com","name":"Epic Games","category":"shop","country":"United States","verdict":"blocked","hostility_score":76,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":36,"bytes":25388,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/store.epicgames.com.json"},{"schema":"ratradar.site/v1","host":"www.gog.com","name":"GOG","category":"shop","country":"Poland","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","A browser load called JSON endpoints that look like data, for example api.gog.com/user/accessToken.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":6,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["api.gog.com/user/accessToken.json","menu.gog.com/v1/account/basic","www.gog.com/cart","menu.gog.com/v1/store/configuration","sections.gog.com/v1/pages/2f/sections/10224066-30ca-408c-baa0-efed01aad9c9"],"inner_page":null},"performance":{"response_ms":1049,"bytes":400000,"redirects":1,"cookies_set":0},"why_this_score":[{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/gog.com.json"},{"schema":"ratradar.site/v1","host":"www.humblebundle.com","name":"Humble Bundle","category":"shop","country":"United States","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.humblebundle.com/sitemap.xml"],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":13,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.humblebundle.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.humblebundle.com/store/warhammer-40000-space-marine-2","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.humblebundle.com/store/image_sitemap.xml"}},"performance":{"response_ms":727,"bytes":400000,"redirects":0,"cookies_set":7},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/humblebundle.com.json"},{"schema":"ratradar.site/v1","host":"www.discogs.com","name":"Discogs","category":"market","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":46,"bytes":5640,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/discogs.com.json"},{"schema":"ratradar.site/v1","host":"www.abebooks.com","name":"AbeBooks","category":"market","country":"Canada","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Read the JSON-LD on the page (WebPage). It is cleaner than the markup.","The site publishes 1 RSS or Atom feed. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example www.abebooks.com/servlet/RecommendationsApi. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"Server"},"robots":{"found":true,"disallow_all":false,"disallow_rules":11,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.abebooks.com/sitemap.adhoc.bdp47.xml.gz","https://www.abebooks.com/sitemap.bdp_index.xml","https://www.abebooks.com/sitemap.plp_index.xml"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":["WebPage"],"feeds":1,"feed_urls":["https://www.abebooks.com/index.rss"],"api_endpoints_seen":["www.abebooks.com/servlet/RecommendationsApi","www.abebooks.com/servlet/ShoppingBasketApi"],"inner_page":null},"performance":{"response_ms":396,"bytes":306441,"redirects":0,"cookies_set":4},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/abebooks.com.json"},{"schema":"ratradar.site/v1","host":"bookshop.org","name":"Bookshop","category":"shop","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Enumerate pages from the sitemap: https://bookshop.org/sitemap.xml"],"risks":["Hostile to automation. Keep volumes low and cache.","robots.txt forbids 12 of 14 known AI crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, ...). An agent that identifies as an AI crawler should not fetch this site.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare","CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":14,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","CCBot","PerplexityBot","Google-Extended","Bytespider","Amazonbot","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":12,"sitemap_urls":["https://bookshop.org/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://bookshop.org/p/books/melissa-doug-pattern-blocks-and-boards-melissa-doug/68447ea2e5119dbc","status":403,"json_ld_types":[],"skipped":null,"sitemap":"https://us-sitemap.bookshop.org/products-0.xml"}},"performance":{"response_ms":359,"bytes":400000,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"CloudFront detected","points":2},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 12 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/bookshop.org.json"},{"schema":"ratradar.site/v1","host":"www.goodreads.com","name":"Goodreads","category":"reviews","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.goodreads.com/siteindex.author.xml"],"risks":["robots.txt forbids some AI crawlers: GPTBot, CCBot."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"Server"},"robots":{"found":true,"disallow_all":false,"disallow_rules":27,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","CCBot"],"ai_crawlers_blocked_count":2,"sitemap_urls":["https://www.goodreads.com/siteindex.author.xml","https://www.goodreads.com/siteindex.author_community_question.xml","https://www.goodreads.com/siteindex.award.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.goodreads.com/questions/6683-why-is-the-publishing-taking-like-1000000","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.goodreads.com/sitemap.366.xml.gz"}},"performance":{"response_ms":463,"bytes":55285,"redirects":0,"cookies_set":4},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 2 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":202,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/goodreads.com.json"},{"schema":"ratradar.site/v1","host":"www.ziprecruiter.com","name":"ZipRecruiter","category":"jobs","country":"United States","verdict":"hostile","hostility_score":50,"recommended_approach":"respect_robots_skip","steps":["robots.txt disallows every crawler. Do not scrape it. Look for an official API, a data license, or a partner feed.","Read the JSON-LD on the page (WebSite, Organization). It is cleaner than the markup."],"risks":["Scraping against an explicit robots.txt ban is the quickest way to a legal or blocking problem.","robots.txt forbids 4 of 14 known AI crawlers (GPTBot, CCBot, Applebot-Extended, Meta-ExternalAgent). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":false,"headless_browser":true,"needs_proxies":false,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":true,"disallow_rules":1,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","CCBot","Applebot-Extended","Meta-ExternalAgent"],"ai_crawlers_blocked_count":4,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":true,"json_ld_types":["WebSite","Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":154,"bytes":108094,"redirects":0,"cookies_set":6},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"robots.txt disallows everything","points":10},{"label":"Raised to at least 50 because robots.txt disallows everything","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt bans every crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/ziprecruiter.com.json"},{"schema":"ratradar.site/v1","host":"www.monster.com","name":"Monster","category":"jobs","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["DataDome","CloudFront"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"CloudFront"},"robots":{"found":true,"disallow_all":false,"disallow_rules":57,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":true,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":108,"bytes":774,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"DataDome detected","points":20},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/monster.com.json"},{"schema":"ratradar.site/v1","host":"www.careerbuilder.com","name":"CareerBuilder","category":"jobs","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["DataDome","CloudFront"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"CloudFront"},"robots":{"found":true,"disallow_all":false,"disallow_rules":40,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":true,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":78,"bytes":780,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"DataDome detected","points":20},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/careerbuilder.com.json"},{"schema":"ratradar.site/v1","host":"www.dice.com","name":"Dice","category":"jobs","country":"United States","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.dice.com/sitemap-index.xml","A browser load called JSON endpoints that look like data, for example cdn-4.convertexperiments.com/api/v1/project-optional-settings/10042981/10044140. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":35,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.dice.com/sitemap-index.xml","https://www.dice.com/hiring/sitemap_index.xml","https://www.dice.com/career-advice/google-news-sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["cdn-4.convertexperiments.com/api/v1/project-optional-settings/10042981/10044140","www.dice.com/services/public/web-config/site-message","www.shareaholic.net/config/5b3c39e0c04501cff97663f3d2320d9f.json","maps.googleapis.com/maps/api/mapsjs/gen_204"],"inner_page":{"url":"https://www.dice.com/career-advice/platform-engineering-vs.-devops-where-should-engineers-specialize-in-2026-and-beyond","status":200,"json_ld_types":["Article"],"skipped":null,"sitemap":"https://www.dice.com/career-advice/google-news-sitemap.xml"}},"performance":{"response_ms":461,"bytes":134497,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/dice.com.json"},{"schema":"ratradar.site/v1","host":"wellfound.com","name":"Wellfound","category":"jobs","country":"United States","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://wellfound.com/sitemap.xml.gz","Read the JSON-LD on the page (Organization, WebSite, WebPage, BreadcrumbList). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example edge.fullstory.com/s/settings/o-1X62V8-na1/v1/web. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":31,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://wellfound.com/sitemap.xml.gz","https://wellfound.com/blog-index.xml.gz"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["Organization","WebSite","WebPage","BreadcrumbList","FAQPage"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["edge.fullstory.com/s/settings/o-1X62V8-na1/v1/web"],"inner_page":{"url":"https://wellfound.com/recruit/products/recruit-pro","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://wellfound.com/basics.xml.gz"}},"performance":{"response_ms":91,"bytes":126894,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/wellfound.com.json"},{"schema":"ratradar.site/v1","host":"remoteok.com","name":"Remote OK","category":"jobs","country":"Netherlands","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","robots.txt asks for a crawl delay of 1 seconds.","Enumerate pages from the sitemap: https://remoteok.com/sitemap.xml","Read the JSON-LD on the page (Organization, Product). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example remoteok.com/assets/all-tags.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":1,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://remoteok.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["Organization","Product"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["remoteok.com/assets/all-tags.json","www.chatbase.co/api/get-chatbot-styles/ZQo7GEuQc49D-cLTpoB5T"],"inner_page":{"url":"https://remoteok.com/remote-jobs/remote-marketing-and-sales-administrator-hunt-st-1137488","status":200,"json_ld_types":["Organization","Product","JobPosting"],"skipped":null,"sitemap":"https://remoteok.com/sitemap-jobs-1.xml"}},"performance":{"response_ms":110,"bytes":59378,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/remoteok.com.json"},{"schema":"ratradar.site/v1","host":"weworkremotely.com","name":"We Work Remotely","category":"jobs","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Enumerate pages from the sitemap: https://weworkremotely.com/sitemap.xml","Read the JSON-LD on the page (Organization, WebSite). It is cleaner than the markup.","The site publishes 1 RSS or Atom feed. Prefer them for new items."],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":7,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://weworkremotely.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["Organization","WebSite"],"feeds":1,"feed_urls":["https://weworkremotely.com/remote-jobs.rss"],"api_endpoints_seen":[],"inner_page":{"url":"https://weworkremotely.com/company/c4media-inc-07724855-233b-4bb5-8aeb-cc099ee10195","status":200,"json_ld_types":["Organization","WebSite"],"skipped":null,"sitemap":"https://weworkremotely.com/sitemap.xml"}},"performance":{"response_ms":690,"bytes":400000,"redirects":0,"cookies_set":4},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/weworkremotely.com.json"},{"schema":"ratradar.site/v1","host":"www.upwork.com","name":"Upwork","category":"jobs","country":"United States","verdict":"blocked","hostility_score":76,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":259,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.upwork.com/sitemap.xml"]},"machine_readable":{"sitemap":false,"llms_txt":true,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":53,"bytes":344241,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/upwork.com.json"},{"schema":"ratradar.site/v1","host":"www.fiverr.com","name":"Fiverr","category":"jobs","country":"Israel","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare","PerimeterX"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":53,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.fiverr.com/sitemap_gigs.xml.gz","https://www.fiverr.com/sitemap_users.xml.gz","https://www.fiverr.com/sitemap_categories.xml.gz"]},"machine_readable":{"sitemap":false,"llms_txt":true,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":97,"bytes":35960,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6},{"label":"PerimeterX detected","points":20}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/fiverr.com.json"},{"schema":"ratradar.site/v1","host":"www.freelancer.com","name":"Freelancer","category":"jobs","country":"Australia","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.freelancer.com/sitemap_index_default_s3.xml","The site publishes 1 RSS or Atom feed. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example www.freelancer.com/ajax-api/site-stats.php. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":38,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.freelancer.com/sitemap_index_default_s3.xml","https://www.freelancer.com/sitemap_index_default_job-search_s3.xml","https://www.freelancer.com/sitemap_index_default_projects_s3.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":1,"feed_urls":["https://www.freelancer.com/rss.xml"],"api_endpoints_seen":["www.freelancer.com/ajax-api/site-stats.php"],"inner_page":{"url":"https://www.freelancer.com/jobs/ts-iso-16949-audit","status":200,"json_ld_types":["Product","Organization","WebSite"],"skipped":null,"sitemap":"https://www.freelancer.com/sitemap/default/jobs-1.xml.gz"}},"performance":{"response_ms":49,"bytes":400000,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/freelancer.com.json"},{"schema":"ratradar.site/v1","host":"www.reed.co.uk","name":"Reed","category":"jobs","country":"United Kingdom","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.reed.co.uk/sitemap.xml","Read the JSON-LD on the page (WebSite, Organization). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.reed.co.uk/api/search/GetRecentSearchesOnlyAutocomplete. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":30,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.reed.co.uk/sitemap.xml","https://www.reed.co.uk/priority-jobseeker-sitemap.xml","https://www.reed.co.uk/courses/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["WebSite","Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.reed.co.uk/api/search/GetRecentSearchesOnlyAutocomplete"],"inner_page":{"url":"https://www.reed.co.uk/courses/free","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.reed.co.uk/courses/sitemap/price.xml"}},"performance":{"response_ms":420,"bytes":132978,"redirects":0,"cookies_set":6},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/reed.co.uk.json"},{"schema":"ratradar.site/v1","host":"www.stepstone.de","name":"StepStone","category":"jobs","country":"Germany","verdict":"open","hostility_score":10,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","A browser load called JSON endpoints that look like data, for example www.stepstone.de/header-footer-bff/candidate/auth. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"API Gateway"},"robots":{"found":true,"disallow_all":false,"disallow_rules":72,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.stepstone.de/header-footer-bff/candidate/auth"],"inner_page":null},"performance":{"response_ms":503,"bytes":300143,"redirects":1,"cookies_set":7},"why_this_score":[{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/stepstone.de.json"},{"schema":"ratradar.site/v1","host":"www.naukri.com","name":"Naukri","category":"jobs","country":"India","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Akamai"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"AkamaiGHost"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":32,"bytes":368,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/naukri.com.json"},{"schema":"ratradar.site/v1","host":"www.jobstreet.com","name":"JobStreet","category":"jobs","country":"Malaysia","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":38,"bytes":5621,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/jobstreet.com.json"},{"schema":"ratradar.site/v1","host":"www.jobbank.gc.ca","name":"Jobbank Canada","category":"jobs","country":"Canada","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","robots.txt asks for a crawl delay of 5 seconds.","A browser load called JSON endpoints that look like data, for example canada.tt.omtrdc.net/rest/v1/delivery. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":5,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["canada.tt.omtrdc.net/rest/v1/delivery","www.jobbank.gc.ca/invitation-manager/config.JSON"],"inner_page":null},"performance":{"response_ms":1772,"bytes":157186,"redirects":1,"cookies_set":3},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/jobbank.gc.ca.json"},{"schema":"ratradar.site/v1","host":"www.usajobs.gov","name":"USAJobs","category":"jobs","country":"United States","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","A browser load called JSON endpoints that look like data, for example www.usajobs.gov/Account/RetrieveUserPreferences. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":4,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.usajobs.gov/sitemap.xml"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.usajobs.gov/Account/RetrieveUserPreferences","www.usajobs.gov/Client/GetClientConfigAsJson"],"inner_page":null},"performance":{"response_ms":63,"bytes":63003,"redirects":0,"cookies_set":1},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/usajobs.gov.json"},{"schema":"ratradar.site/v1","host":"joinhandshake.com","name":"Handshake","category":"jobs","country":"United States","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://joinhandshake.com/sitemap.xml","Read the JSON-LD on the page (WebSite, Organization, ImageObject, WebPage). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example joinhandshake.com/_next/data/ChrUpf1s-76-L9OFPq23U/en.json. Untested, check the terms before using them."],"risks":["robots.txt forbids some AI crawlers: Bytespider."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Vercel","server":"Vercel"},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":["Bytespider"],"ai_crawlers_blocked_count":1,"sitemap_urls":["https://joinhandshake.com/sitemap.xml","https://joinhandshake.com/sitemap-0.xml","https://joinhandshake.com/api/handshake-public-jobs.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebSite","Organization","ImageObject","WebPage"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["joinhandshake.com/_next/data/ChrUpf1s-76-L9OFPq23U/en.json","joinhandshake.com/_next/data/ChrUpf1s-76-L9OFPq23U/en/students.json","joinhandshake.com/_next/data/ChrUpf1s-76-L9OFPq23U/en/career-centers.json","joinhandshake.com/_next/data/ChrUpf1s-76-L9OFPq23U/en/employers.json","joinhandshake.com/_next/data/ChrUpf1s-76-L9OFPq23U/en/research/economic-research.json"],"inner_page":{"url":"https://app.joinhandshake.com/public/jobs/98559","status":200,"json_ld_types":["JobPosting"],"skipped":null,"sitemap":"https://joinhandshake.com/api/handshake-public-jobs/1/"}},"performance":{"response_ms":138,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 1 AI crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/joinhandshake.com.json"},{"schema":"ratradar.site/v1","host":"www.levels.fyi","name":"Levels.fyi","category":"jobs","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","A browser load called JSON endpoints that look like data, for example api.levels.fyi/geo/edge/. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":true,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["api.levels.fyi/geo/edge/","api.levels.fyi/v1/currency","api.levels.fyi/v1/levels/job-family/software-engineer/relevantCompanies"],"inner_page":null},"performance":{"response_ms":88,"bytes":399410,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/levels.fyi.json"},{"schema":"ratradar.site/v1","host":"www.realtor.com","name":"Realtor.com","category":"property","country":"United States","verdict":"blocked","hostility_score":92,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans.","robots.txt forbids some AI crawlers: Meta-ExternalAgent."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Kasada","CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"CloudFront"},"robots":{"found":true,"disallow_all":false,"disallow_rules":285,"crawl_delay_seconds":null,"ai_crawlers_blocked":["Meta-ExternalAgent"],"ai_crawlers_blocked_count":1,"sitemap_urls":["https://www.realtor.com/mortgage/sitemap/mortgagesitemap.xml","https://www.realtor.com/realestateandhomes-detail/sitemap-rss-new/rss-new-index.xml","https://www.realtor.com/realestateandhomes-detail/sitemap-rss-open-house/rss-open-house-index.xml"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":91,"bytes":19591,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"HTTP 429 on a plain request","points":70},{"label":"Kasada detected","points":20},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 1 AI crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":429,"headless_browser_status":429,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/realtor.com.json"},{"schema":"ratradar.site/v1","host":"www.redfin.com","name":"Redfin","category":"property","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Read the JSON-LD on the page (Organization). It is cleaner than the markup.","The site publishes 1 RSS or Atom feed. Prefer them for new items."],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["AWS WAF","CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"istio-envoy"},"robots":{"found":true,"disallow_all":false,"disallow_rules":68,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.redfin.com/newest_listings.xml","https://www.redfin.com/sitemap_com_agentprofiles.xml","https://www.redfin.com/sitemap_com_buildings.xml"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":["Organization"],"feeds":1,"feed_urls":["https://www.redfin.com/newest_listings.rss"],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":225,"bytes":283670,"redirects":0,"cookies_set":10},"why_this_score":[{"label":"AWS WAF detected","points":12},{"label":"CloudFront detected","points":2},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":429,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/redfin.com.json"},{"schema":"ratradar.site/v1","host":"www.trulia.com","name":"Trulia","category":"property","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Enumerate pages from the sitemap: https://www.trulia.com/sitemaps/xml/recent_updates/index.xml","Read the JSON-LD on the page (WebSite). It is cleaner than the markup."],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront","PerimeterX"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"istio-envoy"},"robots":{"found":true,"disallow_all":false,"disallow_rules":138,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.trulia.com/sitemaps/xml/recent_updates/index.xml","https://www.trulia.com/sitemaps/xml/general/index.xml","https://www.trulia.com/sitemaps/xml/newest_for_sale_listings.xml.gz"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.trulia.com/home/0-bollard-rd-columbia-mo-65202-465932550","status":200,"json_ld_types":["RealEstateListing","FAQPage","BreadcrumbList","Product"],"skipped":null,"sitemap":"https://www.trulia.com/sitemaps/xml/newest_for_sale_listings.xml.gz"}},"performance":{"response_ms":329,"bytes":364309,"redirects":0,"cookies_set":3},"why_this_score":[{"label":"CloudFront detected","points":2},{"label":"PerimeterX seen by the browser","points":20},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/trulia.com.json"},{"schema":"ratradar.site/v1","host":"www.apartments.com","name":"Apartments.com","category":"property","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Akamai"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"AkamaiGHost"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":52,"bytes":372,"redirects":0,"cookies_set":3},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/apartments.com.json"},{"schema":"ratradar.site/v1","host":"www.rightmove.co.uk","name":"Rightmove","category":"property","country":"United Kingdom","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.rightmove.co.uk/sitemap.xml"],"risks":["robots.txt forbids some AI crawlers: GPTBot, CCBot."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":93,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","CCBot"],"ai_crawlers_blocked_count":2,"sitemap_urls":["https://www.rightmove.co.uk/sitemap.xml","https://www.rightmove.co.uk/guides/sitemap_index.xml","https://www.rightmove.co.uk/moving-stories/sitemap_index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.rightmove.co.uk/news/articles/property-news/average-asking-prices-in-london-drop-by-over-17000/","status":200,"json_ld_types":["Article","WebPage","BreadcrumbList","WebSite","Person"],"skipped":null,"sitemap":"https://www.rightmove.co.uk/news/post-sitemap.xml"}},"performance":{"response_ms":337,"bytes":337034,"redirects":0,"cookies_set":4},"why_this_score":[{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 2 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/rightmove.co.uk.json"},{"schema":"ratradar.site/v1","host":"www.zoopla.co.uk","name":"Zoopla","category":"property","country":"United Kingdom","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":48,"bytes":5556,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/zoopla.co.uk.json"},{"schema":"ratradar.site/v1","host":"www.immobilienscout24.de","name":"Immobilienscout24","category":"property","country":"Germany","verdict":"blocked","hostility_score":84,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies.","Enumerate pages from the sitemap: https://www.immobilienscout24.de/wissen/sitemap.xml"],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["AWS WAF","CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"CloudFront"},"robots":{"found":true,"disallow_all":false,"disallow_rules":22,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.immobilienscout24.de/wissen/sitemap.xml","https://www.immobilienscout24.de/Suche/sitemap/activeExposes.xml","https://www.immobilienscout24.de/Suche/sitemap/notEmptyPageSearches.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":38,"bytes":4006,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"HTTP 401 on a plain request","points":70},{"label":"AWS WAF detected","points":12},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":401,"headless_browser_status":401,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/immobilienscout24.de.json"},{"schema":"ratradar.site/v1","host":"www.seloger.com","name":"SeLoger","category":"property","country":"France","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Read the JSON-LD on the page (Organization). It is cleaner than the markup."],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront","DataDome"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"CloudFront"},"robots":{"found":true,"disallow_all":false,"disallow_rules":66,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":["Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":255,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"CloudFront detected","points":2},{"label":"DataDome seen by the browser","points":20},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/seloger.com.json"},{"schema":"ratradar.site/v1","host":"www.idealista.com","name":"Idealista","category":"property","country":"Spain","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["DataDome"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"DataDome"},"robots":{"found":true,"disallow_all":false,"disallow_rules":186,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":41,"bytes":776,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"DataDome detected","points":20}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/idealista.com.json"},{"schema":"ratradar.site/v1","host":"www.domain.com.au","name":"Domain","category":"property","country":"Australia","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Akamai"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"AkamaiGHost"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":82,"bytes":373,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/domain.com.au.json"},{"schema":"ratradar.site/v1","host":"www.realestate.com.au","name":"Realestate.com.au","category":"property","country":"Australia","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans.","robots.txt forbids 10 of 14 known AI crawlers (GPTBot, ClaudeBot, anthropic-ai, CCBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Akamai","Kasada","CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":2,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ClaudeBot","anthropic-ai","CCBot","PerplexityBot","Bytespider","Amazonbot","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":10,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":425,"bytes":661,"redirects":0,"cookies_set":4},"why_this_score":[{"label":"HTTP 429 on a plain request","points":70},{"label":"Akamai detected","points":10},{"label":"Kasada detected","points":20},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 10 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":429,"headless_browser_status":429,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/realestate.com.au.json"},{"schema":"ratradar.site/v1","host":"www.zumper.com","name":"Zumper","category":"property","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Read the JSON-LD on the page (WebSite). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.zumper.com/api/t/1/bundle. Untested, check the terms before using them."],"risks":["robots.txt forbids some AI crawlers: CCBot, Bytespider."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":50,"crawl_delay_seconds":null,"ai_crawlers_blocked":["CCBot","Bytespider"],"ai_crawlers_blocked_count":2,"sitemap_urls":["https://www.zumper.com/sitemap.xml.gz"]},"machine_readable":{"sitemap":false,"llms_txt":true,"json_ld_types":["WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.zumper.com/api/t/1/bundle","www.zumper.com/api/t/1/linkpacks/buildings","www.zumper.com/api/t/1/linkpacks/addresses","www.zumper.com/api/svc/inventory/v1/listables/maplist/spotlight"],"inner_page":null},"performance":{"response_ms":115,"bytes":169073,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 2 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/zumper.com.json"},{"schema":"ratradar.site/v1","host":"www.loopnet.com","name":"LoopNet","category":"property","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Akamai"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"AkamaiGHost"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":46,"bytes":369,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/loopnet.com.json"},{"schema":"ratradar.site/v1","host":"rentals.ca","name":"Rentals.ca","category":"property","country":"Canada","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":20,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://rentals.ca/sitemap-global.xml"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":40,"bytes":5550,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/rentals.ca.json"},{"schema":"ratradar.site/v1","host":"www.expedia.com","name":"Expedia","category":"travel","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies.","A browser load called JSON endpoints that look like data, for example www.expedia.com/dd/js/. Untested, check the terms before using them."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Akamai","DataDome"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"istio-envoy"},"robots":{"found":true,"disallow_all":false,"disallow_rules":52,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":true,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.expedia.com/dd/js/"],"inner_page":null},"performance":{"response_ms":172,"bytes":45622,"redirects":0,"cookies_set":13},"why_this_score":[{"label":"HTTP 429 on a plain request","points":70},{"label":"Akamai detected","points":10},{"label":"DataDome seen by the browser","points":20}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":429,"headless_browser_status":429,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/expedia.com.json"},{"schema":"ratradar.site/v1","host":"www.tripadvisor.com","name":"TripAdvisor","category":"travel","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Enumerate pages from the sitemap: https://www.tripadvisor.com/sitemap/2/en_US/sitemap_en_US_index.xml","Read the JSON-LD on the page (Organization, WebSite, LocalBusiness). It is cleaner than the markup."],"risks":["Hostile to automation. Keep volumes low and cache.","robots.txt forbids 9 of 14 known AI crawlers (GPTBot, ClaudeBot, CCBot, Google-Extended, ...). An agent that identifies as an AI crawler should not fetch this site.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai","DataDome","CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":650,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ClaudeBot","CCBot","Google-Extended","Bytespider","Amazonbot","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":9,"sitemap_urls":["https://www.tripadvisor.com/sitemap/2/en_US/sitemap_en_US_index.xml","https://www.tripadvisor.com/sitemap/2/en_US/sitemap_en_US_location_photo_direct_link_index.xml","https://www.tripadvisor.com/sitemap/2/en_US/sitemap_en_US_show_user_reviews_index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["Organization","WebSite","LocalBusiness"],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.tripadvisor.com/AttractionProductReview-g187310-d17405782-Discover_Nuremberg_Medieval_Old_Town_Historic_Markets_Durer-Nuremberg_Middle_Franc.html","status":200,"json_ld_types":["Organization","WebSite","BreadcrumbList","Product","WebPage"],"skipped":null,"sitemap":"https://www.tripadvisor.com/sitemap/att/en_US/sitemap-2199933-en_US-attraction_product_review-1791371135.xml.gz"}},"performance":{"response_ms":561,"bytes":400000,"redirects":0,"cookies_set":4},"why_this_score":[{"label":"Akamai detected","points":10},{"label":"DataDome detected","points":20},{"label":"CloudFront detected","points":2},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 9 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/tripadvisor.com.json"},{"schema":"ratradar.site/v1","host":"www.kayak.com","name":"Kayak","category":"travel","country":"United States","verdict":"open","hostility_score":4,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.kayak.com/sitemap.xml","Read the JSON-LD on the page (Website, Organization, FAQPage). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.kayak.com/s/run/fpc/context. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly","CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":"KAYAK/1.0"},"robots":{"found":true,"disallow_all":false,"disallow_rules":115,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.kayak.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["Website","Organization","FAQPage"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.kayak.com/s/run/fpc/context","www.kayak.com/mvm/p13n/flight/destinations","www.kayak.com/s/horizon/react/component/CurrencyPickerStateProviderAction","www.kayak.com/s/horizon/common/privacy/providers/CookiesDataProvider","www.kayak.com/a/api/smarty/nearby"],"inner_page":{"url":"https://www.kayak.com/hotels","status":200,"json_ld_types":["FAQPage","Website","Organization"],"skipped":null,"sitemap":"https://www.kayak.com/sitemaps/kayak-us-hotel_frontdoor_unknown.xml.gz"}},"performance":{"response_ms":999,"bytes":400000,"redirects":0,"cookies_set":10},"why_this_score":[{"label":"Fastly detected","points":2},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/kayak.com.json"},{"schema":"ratradar.site/v1","host":"www.skyscanner.net","name":"Skyscanner","category":"travel","country":"United Kingdom","verdict":"guarded","hostility_score":37,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls.","Enumerate pages from the sitemap: https://www.skyscanner.net/sitemap.xml","Read the JSON-LD on the page (WebSite). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.skyscanner.net/rf8vapwA/xhr/assets/js/bundle. Untested, check the terms before using them."],"risks":["robots.txt forbids some AI crawlers: Meta-ExternalAgent."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront","PerimeterX"],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"istio-envoy"},"robots":{"found":true,"disallow_all":false,"disallow_rules":324,"crawl_delay_seconds":null,"ai_crawlers_blocked":["Meta-ExternalAgent"],"ai_crawlers_blocked_count":1,"sitemap_urls":["https://www.skyscanner.net/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.skyscanner.net/rf8vapwA/xhr/assets/js/bundle"],"inner_page":{"url":"https://www.skyscanner.net/hotels/afghanistan/ghumay-hotels/ci-45274480","status":200,"json_ld_types":["ItemList"],"skipped":null,"sitemap":"https://www.skyscanner.net/en-gb_hotels-in-city_1_sitemap.xml"}},"performance":{"response_ms":899,"bytes":265733,"redirects":1,"cookies_set":12},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15},{"label":"CloudFront detected","points":2},{"label":"PerimeterX seen by the browser","points":20}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 1 AI crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/skyscanner.net.json"},{"schema":"ratradar.site/v1","host":"www.hotels.com","name":"Hotels.com","category":"travel","country":"United States","verdict":"guarded","hostility_score":45,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":15045,"bytes":null,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"No answer to a plain request","points":40},{"label":"Slow response (15045 ms)","points":5}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":0,"headless_browser_status":0,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/hotels.com.json"},{"schema":"ratradar.site/v1","host":"www.agoda.com","name":"Agoda","category":"travel","country":"Singapore","verdict":"guarded","hostility_score":25,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls.","Enumerate pages from the sitemap: https://www.agoda.com/sitemaps.xml","A browser load called JSON endpoints that look like data, for example www.agoda.com/api/card/campaigns. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":36,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.agoda.com/sitemaps.xml","https://www.agoda.com/fr-fr/sitemaps.xml","https://www.agoda.com/de-de/sitemaps.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.agoda.com/api/card/campaigns","www.agoda.com/api/cronos/layout/GetHotCities","www.agoda.com/api/cronos/seo/base","www.agoda.com/api/cusco/popup/campaigns","www.agoda.com/api/gw/Cms/Lookup","www.agoda.com/api/cronos/home/GetTopDestinations"],"inner_page":{"url":"https://www.agoda.com/africa/malawi/salima.html","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.agoda.com/sitemaps-wl_1_lang_1_pagetype_5_page_1.xml"}},"performance":{"response_ms":211,"bytes":235197,"redirects":0,"cookies_set":14},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15},{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/agoda.com.json"},{"schema":"ratradar.site/v1","host":"www.trip.com","name":"Trip.com","category":"travel","country":"China","verdict":"open","hostility_score":10,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Read the JSON-LD on the page (Organization, TravelAgency, WebSite, SiteNavigationElement). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.trip.com/restapi/soa2/18088/getAppConfig.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":88,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":true,"json_ld_types":["Organization","TravelAgency","WebSite","SiteNavigationElement"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.trip.com/restapi/soa2/18088/getAppConfig.json","us.trip.com/restapi/soa2/13909/getUspInfo","us.trip.com/m/home/getHeaderInfo","us.trip.com/m/home/getOnlineLocales","us.trip.com/m/home/getOnlineCobrandingConfig","us.trip.com/restapi/soa2/24884/json/getConfiguration"],"inner_page":null},"performance":{"response_ms":715,"bytes":143751,"redirects":1,"cookies_set":18},"why_this_score":[{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/trip.com.json"},{"schema":"ratradar.site/v1","host":"www.vrbo.com","name":"Vrbo","category":"travel","country":"United States","verdict":"blocked","hostility_score":80,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies.","A browser load called JSON endpoints that look like data, for example www.vrbo.com/fMkl2kQvX0/GOai/IrqnV_/wiaESpc85VXSwb/ZXFESA/HSkMM2h/5S0c. Untested, check the terms before using them."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":51,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.vrbo.com/fMkl2kQvX0/GOai/IrqnV_/wiaESpc85VXSwb/ZXFESA/HSkMM2h/5S0c"],"inner_page":null},"performance":{"response_ms":191,"bytes":352,"redirects":0,"cookies_set":16},"why_this_score":[{"label":"HTTP 429 on a plain request","points":70},{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":429,"headless_browser_status":429,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/vrbo.com.json"},{"schema":"ratradar.site/v1","host":"www.hostelworld.com","name":"Hostelworld","category":"travel","country":"Ireland","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.hostelworld.com/sitemap-index.xml","Read the JSON-LD on the page (WebSite, WebPage, Organization, SoftwareApplication). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.hostelworld.com/_i18n/99f070da/en/messages.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"Google Frontend"},"robots":{"found":true,"disallow_all":false,"disallow_rules":38,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.hostelworld.com/sitemap-index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["WebSite","WebPage","Organization","SoftwareApplication","MobileApplication"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.hostelworld.com/_i18n/99f070da/en/messages.json","www.hostelworld.com/api/social/global-stats","api.hostelworld.com/static-pages-api/1.0/city/83/","api.sprig.com/sdk/1/environments/FxSIrzlKR6s6/config"],"inner_page":{"url":"https://www.hostelworld.com/hotels/europe/albania/","status":200,"json_ld_types":["WebSite","WebPage","Organization","SoftwareApplication","MobileApplication","BreadcrumbList","Country"],"skipped":null,"sitemap":"https://www.hostelworld.com/sitemap/sitemap-1.xml.gz"}},"performance":{"response_ms":1010,"bytes":305116,"redirects":0,"cookies_set":4},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/hostelworld.com.json"},{"schema":"ratradar.site/v1","host":"www.getyourguide.com","name":"GetYourGuide","category":"travel","country":"Germany","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies.","Enumerate pages from the sitemap: https://www.getyourguide.com/sitemap.xml"],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":33,"bytes":5581,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/getyourguide.com.json"},{"schema":"ratradar.site/v1","host":"www.viator.com","name":"Viator","category":"travel","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["DataDome","Fastly"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":"DataDome"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":29,"bytes":780,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"DataDome detected","points":20},{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/viator.com.json"},{"schema":"ratradar.site/v1","host":"www.delta.com","name":"Delta","category":"travel","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls.","Enumerate pages from the sitemap: https://www.delta.com/sitemap.xml"],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"AmazonS3"},"robots":{"found":true,"disallow_all":false,"disallow_rules":156,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.delta.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.delta.com/PCCOciWeb/app/index.html#/landing","status":null,"json_ld_types":[],"skipped":"robots.txt disallows this path","sitemap":"https://www.delta.com/sitemap.xml"}},"performance":{"response_ms":820,"bytes":8335,"redirects":0,"cookies_set":8},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15},{"label":"Akamai detected","points":10},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":444,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/delta.com.json"},{"schema":"ratradar.site/v1","host":"www.united.com","name":"United","category":"travel","country":"United States","verdict":"guarded","hostility_score":45,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":15040,"bytes":null,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"No answer to a plain request","points":40},{"label":"Slow response (15040 ms)","points":5}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":0,"headless_browser_status":0,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/united.com.json"},{"schema":"ratradar.site/v1","host":"www.aa.com","name":"American Airlines","category":"travel","country":"United States","verdict":"blocked","hostility_score":95,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":[],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":18,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.aa.com/sitemap_index-cms.xml"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":53,"bytes":362,"redirects":0,"cookies_set":5},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/aa.com.json"},{"schema":"ratradar.site/v1","host":"www.southwest.com","name":"Southwest","category":"travel","country":"United States","verdict":"guarded","hostility_score":25,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls.","Enumerate pages from the sitemap: https://www.southwest.com/sitemap.xml","A browser load called JSON endpoints that look like data, for example www.southwest.com/landing/home-page/version.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":2,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.southwest.com/sitemap.xml","https://www.southwest.com/en/flights/sitemap_en.xml","https://espanol.southwest.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.southwest.com/landing/home-page/version.json","www.southwest.com/api/security/v4/security/digital/jwks","soptimize.southwest.com/rest/v1/delivery","www.southwest.com/api/content-delivery/v1/content-delivery/query/placements"],"inner_page":{"url":"https://www.southwest.com/rapid-rewards/partners/hotels/","status":200,"json_ld_types":["Airline","WebSite","MemberProgram","WebPage"],"skipped":null,"sitemap":"https://www.southwest.com/sitemap.xml"}},"performance":{"response_ms":43,"bytes":7551,"redirects":0,"cookies_set":4},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15},{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/southwest.com.json"},{"schema":"ratradar.site/v1","host":"www.ryanair.com","name":"Ryanair","category":"travel","country":"Ireland","verdict":"open","hostility_score":23,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls.","Enumerate pages from the sitemap: https://www.ryanair.com/sitemap.xml","A browser load called JSON endpoints that look like data, for example www.ryanair.com/ryanair3/data/footer/desktop.en-us.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare","CloudFront"],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"AmazonS3"},"robots":{"found":true,"disallow_all":false,"disallow_rules":25,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.ryanair.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.ryanair.com/ryanair3/data/footer/desktop.en-us.json","www.ryanair.com/ryanair3/data/global-header/desktop.en-us.json","www.ryanair.com/ryanair3/data/home-page/desktop.en-us.json","www.ryanair.com/apps/ryanair/i18n.frontend.a11y.accessibility.auth.common-header.customer-portal.farefinder.flight-search.home-page.input-desktop.legalfooter.meta.networkerrors.passengers-selector.side-menu.verified-seal.en-us.json"],"inner_page":{"url":"https://www.ryanair.com/at/de/nutzliche-infos/service-center/allgemeine-geschaftsbedingungen/termsandconditionsar_1579950624","status":200,"json_ld_types":["Organization"],"skipped":null,"sitemap":"https://www.ryanair.com/at/de/content/0/sitemap.xml"}},"performance":{"response_ms":72,"bytes":4743,"redirects":1,"cookies_set":4},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15},{"label":"Cloudflare detected","points":6},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/ryanair.com.json"},{"schema":"ratradar.site/v1","host":"www.lufthansa.com","name":"Lufthansa","category":"travel","country":"Germany","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":2,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.lufthansa.com/sitemap/sitemap_lh.xml.gz","https://www.lufthansa.com/aircore/sitemaps/sitemap-index.xml"]},"machine_readable":{"sitemap":false,"llms_txt":true,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":37,"bytes":5600,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/lufthansa.com.json"},{"schema":"ratradar.site/v1","host":"www.emirates.com","name":"Emirates","category":"travel","country":"United Arab Emirates","verdict":"hostile","hostility_score":55,"recommended_approach":"real_browser","steps":["Use a rendering browser, slow requests, and rotating proxies.","A browser load called JSON endpoints that look like data, for example tr.emirates.com/v1/p. Untested, check the terms before using them."],"risks":["Hostile to automation. Keep volumes low and cache."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["tr.emirates.com/v1/p","tr.emirates.com/v1/i"],"inner_page":null},"performance":{"response_ms":15045,"bytes":null,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"No answer to a plain request","points":40},{"label":"Akamai seen by the browser","points":10},{"label":"Slow response (15045 ms)","points":5}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":0,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/emirates.com.json"},{"schema":"ratradar.site/v1","host":"www.uber.com","name":"Uber","category":"travel","country":"United States","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.uber.com/sitemap.xml","A browser load called JSON endpoints that look like data, for example www.uber.com/api/getCurrentUser. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":27,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.uber.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.uber.com/api/getCurrentUser","www.uber.com/bootstrap.json","www.uber.com/api/getExperiments","www.uber.com/api/getMerchandisingAdAvailability","www.uber.com/api/getProductSuggestions","www.uber.com/api/getMapHeroEnabledProducts"],"inner_page":{"url":"https://www.uber.com/us/zh/e/drive/evening-jobs/corvallis-or-us/","status":200,"json_ld_types":["JobPosting","FAQPage"],"skipped":null,"sitemap":"https://www.uber.com/dynamic_drive_evening-jobs_narrowgeo-sitemap.xml"}},"performance":{"response_ms":425,"bytes":400000,"redirects":1,"cookies_set":5},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/uber.com.json"},{"schema":"ratradar.site/v1","host":"www.lyft.com","name":"Lyft","category":"travel","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.lyft.com/sitemap.xml","Read the JSON-LD on the page (Organization). It is cleaner than the markup."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"envoy-iad"},"robots":{"found":true,"disallow_all":false,"disallow_rules":31,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.lyft.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.lyft.com/blog/posts/lyft-supports-local-nonprofits-with-more-than-150000-in-transportation-grants","status":200,"json_ld_types":["NewsArticle"],"skipped":null,"sitemap":"https://cdn.lyft.com/lyftcomfe/sitemap-blog.xml"}},"performance":{"response_ms":490,"bytes":400000,"redirects":0,"cookies_set":3},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/lyft.com.json"},{"schema":"ratradar.site/v1","host":"www.grab.com","name":"Grab","category":"travel","country":"Singapore","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.grab.com/sg/sitemap.xml","Read the JSON-LD on the page (FAQPage, Organization). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example s.yimg.com/wi/config/10022897.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":70,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.grab.com/sg/sitemap.xml","https://www.grab.com/my/sitemap.xml","https://www.grab.com/ph/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["FAQPage","Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["s.yimg.com/wi/config/10022897.json"],"inner_page":{"url":"https://www.grab.com/sg/inside-grab/stories/how-grab-users-helped-plant-200000-trees-in-thailand/","status":200,"json_ld_types":["Organization"],"skipped":null,"sitemap":"https://www.grab.com/sg/sitemap.xml"}},"performance":{"response_ms":762,"bytes":400000,"redirects":1,"cookies_set":2},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/grab.com.json"},{"schema":"ratradar.site/v1","host":"www.amtrak.com","name":"Amtrak","category":"travel","country":"United States","verdict":"open","hostility_score":16,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.amtrak.com/sitemap.xml","A browser load called JSON endpoints that look like data, for example www.amtrak.com/libs/granite/csrf/token.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai","Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":15,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.amtrak.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.amtrak.com/libs/granite/csrf/token.json","www.amtrak.com/6urN4M5Vz0EJNvY0_Fn7/wOYfNLGwpwLXhLOD/Gwx5YCEoDgg/cXlzYm1/9fCQ","www.amtrak.com/services/staticmappingjson"],"inner_page":{"url":"https://www.amtrak.com/guestrewards/account-overview/missing-points","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.amtrak.com/sitemap.xml"}},"performance":{"response_ms":97,"bytes":239362,"redirects":1,"cookies_set":4},"why_this_score":[{"label":"Akamai detected","points":10},{"label":"Cloudflare seen by the browser","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/amtrak.com.json"},{"schema":"ratradar.site/v1","host":"www.eurail.com","name":"Eurail","category":"travel","country":"Netherlands","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Read the JSON-LD on the page (Organization). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.interrail.com/bin/countries. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"CloudFront"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":["Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.interrail.com/bin/countries","www.interrail.com/bin/regional-settings.json","www.google.com/ccm/geo","www.interrail.com/bin/miniCartConfiguration"],"inner_page":null},"performance":{"response_ms":556,"bytes":137317,"redirects":1,"cookies_set":2},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/eurail.com.json"},{"schema":"ratradar.site/v1","host":"www.tiktok.com","name":"TikTok","category":"social","country":"United States","verdict":"guarded","hostility_score":31,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls.","Enumerate pages from the sitemap: https://www.tiktok.com/sitemap.xml","A browser load called JSON endpoints that look like data, for example mon16-normal-useast5.tiktokv.us/monitor_web/settings/browser-settings. Untested, check the terms before using them."],"risks":["robots.txt forbids 11 of 14 known AI crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare","Akamai"],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":16,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Bytespider","Applebot-Extended","Meta-ExternalAgent"],"ai_crawlers_blocked_count":11,"sitemap_urls":[]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["mon16-normal-useast5.tiktokv.us/monitor_web/settings/browser-settings","mcs-ttp2.tiktokv.us/v1/list","mcs.tiktokw.us/v1/user/webid","www.tiktok.com/tiktok/pns/privacy_headers","www.tiktok.com/tiktok/popup/dispatch/v1","mcs.tiktokw.us/v1/list"],"inner_page":{"url":"https://www.tiktok.com/@dearmebeauty/video/6893431881816149250","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.tiktok.com/sitemap.xml"}},"performance":{"response_ms":308,"bytes":367990,"redirects":0,"cookies_set":3},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15},{"label":"Cloudflare detected","points":6},{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 11 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/tiktok.com.json"},{"schema":"ratradar.site/v1","host":"www.pinterest.com","name":"Pinterest","category":"social","country":"United States","verdict":"hostile","hostility_score":50,"recommended_approach":"respect_robots_skip","steps":["robots.txt disallows every crawler. Do not scrape it. Look for an official API, a data license, or a partner feed.","A browser load called JSON endpoints that look like data, for example www.pinterest.com/secure/unauth_user_data/. Untested, check the terms before using them."],"risks":["Scraping against an explicit robots.txt ban is the quickest way to a legal or blocking problem."],"can_scrape":{"plain_http":false,"headless_browser":true,"needs_proxies":false,"official_api_advised":true},"protections":{"vendors":["Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":true,"disallow_rules":1,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261205_first_day_of_hanukkah_www.pinterest.com.xml","https://www.pinterest.com/v3_sitemaps/ideas_hub_expansion_v2_sitemap_www.pinterest.com.xml","https://www.pinterest.com/v3_sitemaps/seasonal_events_sitemap_20261201_freedom_and_democracy_day_www.pinterest.com.xml"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.pinterest.com/secure/unauth_user_data/","www.pinterest.com/secure/arkose_data_exchange/","www.pinterest.com/resource/AdvancedTypeaheadResource/get/","www.pinterest.com/resource/SearchTabResource/get/","www.pinterest.com/resource/ApiResource/get/","www.pinterest.com/resource/UserExperienceResource/get/"],"inner_page":null},"performance":{"response_ms":401,"bytes":400000,"redirects":0,"cookies_set":4},"why_this_score":[{"label":"Fastly detected","points":2},{"label":"robots.txt disallows everything","points":10},{"label":"Raised to at least 50 because robots.txt disallows everything","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt bans every crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/pinterest.com.json"},{"schema":"ratradar.site/v1","host":"www.snapchat.com","name":"Snapchat","category":"social","country":"United States","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.snapchat.com/sitemap.xml"],"risks":["robots.txt forbids 11 of 14 known AI crawlers (GPTBot, ClaudeBot, Claude-Web, anthropic-ai, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"API Gateway"},"robots":{"found":true,"disallow_all":false,"disallow_rules":4,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ClaudeBot","Claude-Web","anthropic-ai","CCBot","Google-Extended","Bytespider","Amazonbot","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":11,"sitemap_urls":["https://www.snapchat.com/sitemap.xml","https://www.snapchat.com/lens/sitemap.xml","https://www.snapchat.com/sitemaps/place/sitemap-index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.snapchat.com/community/onboarding","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.snapchat.com/sitemap-0.xml"}},"performance":{"response_ms":38,"bytes":130024,"redirects":0,"cookies_set":0},"why_this_score":[],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 11 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/snapchat.com.json"},{"schema":"ratradar.site/v1","host":"www.tumblr.com","name":"Tumblr","category":"social","country":"United States","verdict":"open","hostility_score":17,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls.","robots.txt asks for a crawl delay of 1 seconds.","Enumerate pages from the sitemap: https://www.tumblr.com/sitemap.xml","A browser load called JSON endpoints that look like data, for example www.tumblr.com/services/bblog. Untested, check the terms before using them."],"risks":["robots.txt forbids 7 of 14 known AI crawlers (ClaudeBot, anthropic-ai, CCBot, Google-Extended, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":20,"crawl_delay_seconds":1,"ai_crawlers_blocked":["ClaudeBot","anthropic-ai","CCBot","Google-Extended","Amazonbot","Applebot-Extended","Meta-ExternalAgent"],"ai_crawlers_blocked_count":7,"sitemap_urls":["https://www.tumblr.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.tumblr.com/services/bblog","www.tumblr.com/svc/pop/config","www.tumblr.com/api/v2/explore/trending/blogs","www.tumblr.com/api/v2/explore/trending/tags","www.tumblr.com/api/v2/explore/trending"],"inner_page":{"url":"https://www.tumblr.com/explore/staff-picks","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.tumblr.com/sitemap-static.xml"}},"performance":{"response_ms":255,"bytes":117086,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 7 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/tumblr.com.json"},{"schema":"ratradar.site/v1","host":"www.quora.com","name":"Quora","category":"social","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"respect_robots_skip","steps":["robots.txt disallows every crawler. Do not scrape it. Look for an official API, a data license, or a partner feed."],"risks":["Scraping against an explicit robots.txt ban is the quickest way to a legal or blocking problem.","robots.txt forbids 7 of 14 known AI crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":false,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":true,"disallow_rules":63,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","PerplexityBot","Bytespider","Applebot-Extended"],"ai_crawlers_blocked_count":7,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":31,"bytes":5574,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6},{"label":"robots.txt disallows everything","points":10},{"label":"Raised to at least 50 because robots.txt disallows everything","points":0}],"eu_ai_training":{"level":"strong","signals":[{"kind":"robots_txt","label":"robots.txt bans every crawler"},{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/quora.com.json"},{"schema":"ratradar.site/v1","host":"medium.com","name":"Medium","category":"social","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies.","Enumerate pages from the sitemap: https://medium.com/sitemap/sitemap.xml"],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans.","robots.txt forbids 6 of 14 known AI crawlers (GPTBot, ClaudeBot, Bytespider, Amazonbot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":15,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ClaudeBot","Bytespider","Amazonbot","Applebot-Extended","Meta-ExternalAgent"],"ai_crawlers_blocked_count":6,"sitemap_urls":["https://medium.com/sitemap/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":26,"bytes":5550,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"strong","signals":[{"kind":"robots_txt","label":"robots.txt blocks 6 AI crawlers"},{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/medium.com.json"},{"schema":"ratradar.site/v1","host":"discord.com","name":"Discord","category":"social","country":"United States","verdict":"open","hostility_score":8,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://discord.com/servers/servers-sitemap-index.xml","Read the JSON-LD on the page (WebSite, WebPage, Organization). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example discord.com/api/v9/users/@me. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare","CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":24,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://discord.com/servers/servers-sitemap-index.xml","https://discord.com/servers/servers-parent-search-sitemap-index.xml","https://discord.com/application-directory-seo/sitemap-index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebSite","WebPage","Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["discord.com/api/v9/users/@me","discord.com/api/v9/experiments","discord.com/api/v9/auth/location-metadata","discord.com/api/v9/apex/experiments"],"inner_page":{"url":"https://discord.com/discovery/applications/1464436046195593387","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://discord.com/application-directory-seo/sitemap.xml"}},"performance":{"response_ms":61,"bytes":170374,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/discord.com.json"},{"schema":"ratradar.site/v1","host":"www.twitch.tv","name":"Twitch","category":"media","country":"United States","verdict":"open","hostility_score":15,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls.","Read the JSON-LD on the page (Organization). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example gql.twitch.tv/gql. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":43,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.twitch.tv/sitemapv2_index.xml.gz"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":["Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["gql.twitch.tv/gql","assets.twitch.tv/config/manifest.json"],"inner_page":null},"performance":{"response_ms":62,"bytes":203359,"redirects":0,"cookies_set":4},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/twitch.tv.json"},{"schema":"ratradar.site/v1","host":"mastodon.social","name":"Mastodon","category":"social","country":"Germany","verdict":"open","hostility_score":17,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls.","A browser load called JSON endpoints that look like data, for example mastodon.social/packs/assets/iamcal-DP8KB4yM.json. Untested, check the terms before using them."],"risks":["robots.txt forbids some AI crawlers: GPTBot."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly"],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":"Mastodon"},"robots":{"found":true,"disallow_all":false,"disallow_rules":3,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot"],"ai_crawlers_blocked_count":1,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["mastodon.social/packs/assets/iamcal-DP8KB4yM.json","mastodon.social/api/v1/custom_emojis","mastodon.social/packs/assets/compact-NMaGTNYN.json","mastodon.social/packs/assets/cldr-C1i5KVEL.json","mastodon.social/api/v2/instance","mastodon.social/api/v1/trends/tags"],"inner_page":null},"performance":{"response_ms":475,"bytes":51258,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15},{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 1 AI crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/mastodon.social.json"},{"schema":"ratradar.site/v1","host":"bsky.app","name":"Bluesky","category":"social","country":"United States","verdict":"open","hostility_score":15,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls.","A browser load called JSON endpoints that look like data, for example events.bsky.app/gb/api/features/sdk-7gkUkGy9wguUjyFe. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":1,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://bsky.app/sitemap/users.xml.gz"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["events.bsky.app/gb/api/features/sdk-7gkUkGy9wguUjyFe","ip.bsky.app/geolocation","live-events.workers.bsky.app/config","app-config.workers.bsky.app/config","public.api.bsky.app/xrpc/app.bsky.ageassurance.getConfig","public.api.bsky.app/xrpc/app.bsky.unspecced.getConfig"],"inner_page":null},"performance":{"response_ms":312,"bytes":24220,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/bsky.app.json"},{"schema":"ratradar.site/v1","host":"www.threads.net","name":"Threads","category":"social","country":"United States","verdict":"hostile","hostility_score":50,"recommended_approach":"respect_robots_skip","steps":["robots.txt disallows every crawler. Do not scrape it. Look for an official API, a data license, or a partner feed."],"risks":["Scraping against an explicit robots.txt ban is the quickest way to a legal or blocking problem.","robots.txt forbids 6 of 14 known AI crawlers (GPTBot, ClaudeBot, PerplexityBot, Google-Extended, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":false,"headless_browser":true,"needs_proxies":false,"official_api_advised":true},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":true,"disallow_rules":1,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ClaudeBot","PerplexityBot","Google-Extended","Amazonbot","Applebot-Extended"],"ai_crawlers_blocked_count":6,"sitemap_urls":["https://www.threads.com/sitemap/threads_profiles_sitemap.xml.gz","https://www.threads.com/sitemap/threads_profiles_sitemap_tier_1.xml.gz","https://www.threads.com/sitemap/threads_profiles_sitemap_tier_2.xml.gz"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":392,"bytes":400000,"redirects":1,"cookies_set":1},"why_this_score":[{"label":"robots.txt disallows everything","points":10},{"label":"Raised to at least 50 because robots.txt disallows everything","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt bans every crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/threads.net.json"},{"schema":"ratradar.site/v1","host":"vk.com","name":"VK","category":"social","country":"Russia","verdict":"open","hostility_score":21,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls.","A browser load called JSON endpoints that look like data, for example vk.ru/al_index.php. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"kittenx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":79,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["vk.ru/al_index.php","web.api.vk.ru/method/utils.getStaticVersions","login.vk.ru/","web.api.vk.ru/method/account.getInfo","api.vk.ru/method/auth.getAnonymToken"],"inner_page":null},"performance":{"response_ms":1317,"bytes":207645,"redirects":0,"cookies_set":7},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/vk.com.json"},{"schema":"ratradar.site/v1","host":"weibo.com","name":"Weibo","category":"social","country":"China","verdict":"hostile","hostility_score":50,"recommended_approach":"respect_robots_skip","steps":["robots.txt disallows every crawler. Do not scrape it. Look for an official API, a data license, or a partner feed.","robots.txt asks for a crawl delay of 1 seconds.","Enumerate pages from the sitemap: https://weibo.com/sitemap.xml","A browser load called JSON endpoints that look like data, for example passport.weibo.com/sso/bd. Untested, check the terms before using them."],"risks":["Scraping against an explicit robots.txt ban is the quickest way to a legal or blocking problem.","robots.txt forbids 4 of 14 known AI crawlers (ChatGPT-User, OAI-SearchBot, Bytespider, Meta-ExternalAgent). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":false,"official_api_advised":true},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"APISIX/3.14.1"},"robots":{"found":true,"disallow_all":true,"disallow_rules":1,"crawl_delay_seconds":1,"ai_crawlers_blocked":["ChatGPT-User","OAI-SearchBot","Bytespider","Meta-ExternalAgent"],"ai_crawlers_blocked_count":4,"sitemap_urls":["https://weibo.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["passport.weibo.com/sso/bd"],"inner_page":null},"performance":{"response_ms":1515,"bytes":9486,"redirects":1,"cookies_set":0},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15},{"label":"robots.txt disallows everything","points":10},{"label":"Raised to at least 50 because robots.txt disallows everything","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt bans every crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/weibo.com.json"},{"schema":"ratradar.site/v1","host":"www.douban.com","name":"Douban","category":"social","country":"China","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.douban.com/sitemap_index.xml"],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"dae"},"robots":{"found":true,"disallow_all":false,"disallow_rules":20,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.douban.com/sitemap_index.xml","https://www.douban.com/sitemap_updated_index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.douban.com/group/topic/350248899/","status":403,"json_ld_types":[],"skipped":null,"sitemap":"https://www.douban.com/sitemap.xml.gz"}},"performance":{"response_ms":1261,"bytes":92144,"redirects":0,"cookies_set":2},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/douban.com.json"},{"schema":"ratradar.site/v1","host":"line.me","name":"LINE","category":"social","country":"Japan","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","A browser load called JSON endpoints that look like data, for example www.line.me/static/locales/en/translation.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"STORAGE"},"robots":{"found":true,"disallow_all":false,"disallow_rules":7,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.line.me/static/locales/en/translation.json"],"inner_page":null},"performance":{"response_ms":771,"bytes":35792,"redirects":1,"cookies_set":0},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/line.me.json"},{"schema":"ratradar.site/v1","host":"www.kakaocorp.com","name":"KakaoTalk","category":"social","country":"South Korea","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.kakaocorp.com/page/sitemap.xml","Read the JSON-LD on the page (WebSite, Organization). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.kakaocorp.com/api/v3/search-bar. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.kakaocorp.com/page/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebSite","Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.kakaocorp.com/api/v3/search-bar","www.kakaocorp.com/api/v3/content/home","www.kakaocorp.com/api/v2/content/footer"],"inner_page":{"url":"https://www.kakaocorp.com/page/detail/11155","status":200,"json_ld_types":["Organization"],"skipped":null,"sitemap":"https://www.kakaocorp.com/page/sitemap.xml"}},"performance":{"response_ms":1705,"bytes":45045,"redirects":1,"cookies_set":1},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/kakaocorp.com.json"},{"schema":"ratradar.site/v1","host":"www.producthunt.com","name":"Product Hunt","category":"social","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","The site publishes 1 RSS or Atom feed. Prefer them for new items."],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":14,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.producthunt.com/sitemaps_v3/root_sitemap.xml.gz","https://www.producthunt.com/sitemaps_v3/stories_sitemap.xml.gz","https://www.producthunt.com/sitemaps_v3/topics_sitemap.xml.gz"]},"machine_readable":{"sitemap":false,"llms_txt":true,"json_ld_types":[],"feeds":1,"feed_urls":["https://www.producthunt.com/feed"],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":63,"bytes":400000,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/producthunt.com.json"},{"schema":"ratradar.site/v1","host":"dev.to","name":"Dev.to","category":"dev","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://dev.to/sitemap-index.xml","The site publishes 1 RSS or Atom feed. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example dev.to/async_info/base_data. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":"Heroku"},"robots":{"found":true,"disallow_all":false,"disallow_rules":16,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://dev.to/sitemap-index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":[],"feeds":1,"feed_urls":["https://dev.to/feed"],"api_endpoints_seen":["dev.to/async_info/base_data","dev.to/ahoy/visits","dev.to/bb_tabulations","jnn-pa.googleapis.com/$rpc/google.internal.waa.v1.Waa/GenerateIT"],"inner_page":{"url":"https://dev.to/pyla_neeraja_7c1a2a759925/naturequest-ai-turn-screen-time-into-outdoor-adventures-5fh0","status":200,"json_ld_types":["Article"],"skipped":null,"sitemap":"https://dev.to/sitemap-posts-0.xml"}},"performance":{"response_ms":81,"bytes":270102,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/dev.to.json"},{"schema":"ratradar.site/v1","host":"lemmy.world","name":"Lemmy","category":"social","country":"Germany","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies.","robots.txt asks for a crawl delay of 60 seconds.","Enumerate pages from the sitemap: https://lemmy.world/sitemap.xml"],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":12,"crawl_delay_seconds":60,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":60,"bytes":5636,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/lemmy.world.json"},{"schema":"ratradar.site/v1","host":"www.meetup.com","name":"Meetup","category":"social","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.meetup.com/pro-index-sitemap.xml","Read the JSON-LD on the page (Organization, Event). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.meetup.com/intl/en-US.31e5d.json. Untested, check the terms before using them."],"risks":["robots.txt forbids some AI crawlers: GPTBot."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":115,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot"],"ai_crawlers_blocked_count":1,"sitemap_urls":["https://www.meetup.com/pro-index-sitemap.xml","https://www.meetup.com/topics-index-sitemap.xml","https://www.meetup.com/topics-localized-index-sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["Organization","Event"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.meetup.com/intl/en-US.31e5d.json","www.meetup.com/gql2","www.meetup.com/OTo8P26K","www.meetup.com/orion/v3/identity/settings"],"inner_page":{"url":"https://www.meetup.com/lawofattractioncentre/events/316075505/","status":200,"json_ld_types":["Organization","BreadcrumbList","Event","RsvpAction"],"skipped":null,"sitemap":"https://www.meetup.com/sw_events_1.xml.gz"}},"performance":{"response_ms":47,"bytes":400000,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 1 AI crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/meetup.com.json"},{"schema":"ratradar.site/v1","host":"nextdoor.com","name":"Nextdoor","category":"social","country":"United States","verdict":"hostile","hostility_score":50,"recommended_approach":"respect_robots_skip","steps":["robots.txt disallows every crawler. Do not scrape it. Look for an official API, a data license, or a partner feed.","Read the JSON-LD on the page (Organization). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example ads.nextdoor.com/v2/api/conversions/settings. Untested, check the terms before using them."],"risks":["Scraping against an explicit robots.txt ban is the quickest way to a legal or blocking problem.","robots.txt forbids some AI crawlers: GPTBot."],"can_scrape":{"plain_http":false,"headless_browser":true,"needs_proxies":false,"official_api_advised":true},"protections":{"vendors":["CloudFront","Cloudflare"],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"istio-envoy"},"robots":{"found":true,"disallow_all":true,"disallow_rules":1,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot"],"ai_crawlers_blocked_count":1,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":["Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["ads.nextdoor.com/v2/api/conversions/settings","s.yimg.com/wi/config/10216759.json"],"inner_page":null},"performance":{"response_ms":140,"bytes":38381,"redirects":0,"cookies_set":4},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15},{"label":"CloudFront detected","points":2},{"label":"Cloudflare seen by the browser","points":6},{"label":"robots.txt disallows everything","points":10},{"label":"Raised to at least 50 because robots.txt disallows everything","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt bans every crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/nextdoor.com.json"},{"schema":"ratradar.site/v1","host":"www.patreon.com","name":"Patreon","category":"media","country":"United States","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.patreon.com/sitemap.xml","A browser load called JSON endpoints that look like data, for example transcend-cdn.com/cm/8dec6ba2-a601-4c04-a3ba-e4d5b45000f4/translations/en.json. Untested, check the terms before using them."],"risks":["robots.txt forbids some AI crawlers: PerplexityBot."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":17,"crawl_delay_seconds":null,"ai_crawlers_blocked":["PerplexityBot"],"ai_crawlers_blocked_count":1,"sitemap_urls":["https://www.patreon.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["transcend-cdn.com/cm/8dec6ba2-a601-4c04-a3ba-e4d5b45000f4/translations/en.json","www.patreon.com/_marketing_assets/_next/data/19muidp5Y1dFVMIMBTTFo/index.json","www.patreon.com/_marketing_assets/_next/data/19muidp5Y1dFVMIMBTTFo/creators/podcasts.json","www.patreon.com/_marketing_assets/assets/data/uil.1791565804711.json"],"inner_page":{"url":"https://www.patreon.com/hatton_slayden/posts/touch-of-1-1768985","status":403,"json_ld_types":[],"skipped":null,"sitemap":"https://www.patreon.com/sitemap_posts_0.xml"}},"performance":{"response_ms":119,"bytes":335203,"redirects":0,"cookies_set":5},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 1 AI crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/patreon.com.json"},{"schema":"ratradar.site/v1","host":"ko-fi.com","name":"Ko-fi","category":"media","country":"United Kingdom","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here."],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":51,"bytes":68463,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/ko-fi.com.json"},{"schema":"ratradar.site/v1","host":"www.cnn.com","name":"CNN","category":"news","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.cnn.com/sitemap/news.xml"],"risks":["robots.txt forbids 13 of 14 known AI crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":36,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","Claude-Web","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Bytespider","Amazonbot","Applebot-Extended","cohere-ai"],"ai_crawlers_blocked_count":13,"sitemap_urls":["https://www.cnn.com/sitemap/news.xml","https://www.cnn.com/sitemap/article/cnn-underscored.xml","https://www.cnn.com/sitemap/section/cnn-underscored.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.cnn.com/2026/10/11/politics/video/a-democrat-has-never-represented-colorados-5th-congressional-district-will-this-be-the-cycle-that-changes","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.cnn.com/sitemap/news.xml"}},"performance":{"response_ms":60,"bytes":400000,"redirects":0,"cookies_set":6},"why_this_score":[{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 13 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/cnn.com.json"},{"schema":"ratradar.site/v1","host":"www.washingtonpost.com","name":"Washington Post","category":"news","country":"United States","verdict":"guarded","hostility_score":45,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429."],"risks":["robots.txt forbids 9 of 14 known AI crawlers (ClaudeBot, Claude-Web, anthropic-ai, CCBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":113,"crawl_delay_seconds":null,"ai_crawlers_blocked":["ClaudeBot","Claude-Web","anthropic-ai","CCBot","PerplexityBot","Bytespider","Amazonbot","Applebot-Extended","Meta-ExternalAgent"],"ai_crawlers_blocked_count":9,"sitemap_urls":["https://www.washingtonpost.com/sitemaps/sitemap.xml.gz","https://www.washingtonpost.com/sitemaps/news-sitemap.xml.gz","https://www.washingtonpost.com/sitemaps/author-sitemap.xml.gz"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":15042,"bytes":null,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"No answer to a plain request","points":40},{"label":"Slow response (15042 ms)","points":5}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 9 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":0,"headless_browser_status":0,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/washingtonpost.com.json"},{"schema":"ratradar.site/v1","host":"www.wsj.com","name":"Wall Street Journal","category":"news","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"respect_robots_skip","steps":["robots.txt disallows every crawler. Do not scrape it. Look for an official API, a data license, or a partner feed.","Enumerate pages from the sitemap: https://www.wsj.com/sitemap.xml"],"risks":["Scraping against an explicit robots.txt ban is the quickest way to a legal or blocking problem."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":false,"official_api_advised":true},"protections":{"vendors":["DataDome","CloudFront"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"CloudFront"},"robots":{"found":true,"disallow_all":true,"disallow_rules":1,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.wsj.com/sitemap.xml","https://www.wsj.com/wsjsitemaps/wsj_google_news.xml","https://www.wsj.com/wsj_video_recent.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":54,"bytes":767,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 401 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"DataDome detected","points":20},{"label":"CloudFront detected","points":2},{"label":"robots.txt disallows everything","points":10},{"label":"Raised to at least 50 because robots.txt disallows everything","points":0}],"eu_ai_training":{"level":"strong","signals":[{"kind":"robots_txt","label":"robots.txt bans every crawler"},{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":401,"headless_browser_status":401,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/wsj.com.json"},{"schema":"ratradar.site/v1","host":"www.bloomberg.com","name":"Bloomberg","category":"news","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies.","Enumerate pages from the sitemap: https://www.bloomberg.com/sitemaps/news/index.xml"],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans.","robots.txt forbids 14 of 14 known AI crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["PerimeterX","Fastly"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":["retry-after"],"cdn":"Fastly","server":"Varnish"},"robots":{"found":true,"disallow_all":false,"disallow_rules":25,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","Claude-Web","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Bytespider","Amazonbot","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":14,"sitemap_urls":["https://www.bloomberg.com/sitemaps/news/index.xml","https://www.bloomberg.com/sitemaps/news/latest.xml","https://www.bloomberg.com/sitemaps/collections/index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":96,"bytes":13856,"redirects":0,"cookies_set":6},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"PerimeterX detected","points":20},{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"strong","signals":[{"kind":"robots_txt","label":"robots.txt blocks 14 AI crawlers"},{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/bloomberg.com.json"},{"schema":"ratradar.site/v1","host":"www.reuters.com","name":"Reuters","category":"news","country":"United Kingdom","verdict":"blocked","hostility_score":100,"recommended_approach":"respect_robots_skip","steps":["robots.txt disallows every crawler. Do not scrape it. Look for an official API, a data license, or a partner feed.","Enumerate pages from the sitemap: https://www.reuters.com/arc/outboundfeeds/sitemap-index/?outputType=xml"],"risks":["Scraping against an explicit robots.txt ban is the quickest way to a legal or blocking problem.","robots.txt forbids some AI crawlers: Applebot-Extended."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":false,"official_api_advised":true},"protections":{"vendors":["DataDome","CloudFront"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"CloudFront"},"robots":{"found":true,"disallow_all":true,"disallow_rules":1,"crawl_delay_seconds":null,"ai_crawlers_blocked":["Applebot-Extended"],"ai_crawlers_blocked_count":1,"sitemap_urls":["https://www.reuters.com/arc/outboundfeeds/sitemap-index/?outputType=xml","https://www.reuters.com/arc/outboundfeeds/sitemap-jp-index/?outputType=xml","https://www.reuters.com/arc/outboundfeeds/news-sitemap-index/?outputType=xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":53,"bytes":774,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 401 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"DataDome detected","points":20},{"label":"CloudFront detected","points":2},{"label":"robots.txt disallows everything","points":10},{"label":"Raised to at least 50 because robots.txt disallows everything","points":0}],"eu_ai_training":{"level":"strong","signals":[{"kind":"robots_txt","label":"robots.txt bans every crawler"},{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":401,"headless_browser_status":401,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/reuters.com.json"},{"schema":"ratradar.site/v1","host":"apnews.com","name":"AP News","category":"news","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans.","robots.txt forbids 9 of 14 known AI crawlers (GPTBot, ClaudeBot, Claude-Web, anthropic-ai, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":18,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ClaudeBot","Claude-Web","anthropic-ai","CCBot","PerplexityBot","Amazonbot","Applebot-Extended","cohere-ai"],"ai_crawlers_blocked_count":9,"sitemap_urls":["https://apnews.com/ap-sitemap.xml","https://apnews.com/news-sitemap-content.xml","https://apnews.com/hubs-sitemap-content.xml"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":163,"bytes":5550,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"strong","signals":[{"kind":"robots_txt","label":"robots.txt blocks 9 AI crawlers"},{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/apnews.com.json"},{"schema":"ratradar.site/v1","host":"www.npr.org","name":"NPR","category":"news","country":"United States","verdict":"hostile","hostility_score":51,"recommended_approach":"real_browser","steps":["Use a rendering browser, slow requests, and rotating proxies.","A browser load called JSON endpoints that look like data, for example web-video.ads.aps.amazon-adsystem.com/e/dtb/bid. Untested, check the terms before using them."],"risks":["Hostile to automation. Keep volumes low and cache."],"can_scrape":{"plain_http":false,"headless_browser":true,"needs_proxies":true,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["web-video.ads.aps.amazon-adsystem.com/e/dtb/bid","id.crwdcntrl.net/id"],"inner_page":null},"performance":{"response_ms":15037,"bytes":null,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"No answer to a plain request","points":40},{"label":"Cloudflare seen by the browser","points":6},{"label":"Slow response (15037 ms)","points":5},{"label":"A real browser got in though a plain request did not","points":0}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":0,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/npr.org.json"},{"schema":"ratradar.site/v1","host":"www.foxnews.com","name":"Fox News","category":"news","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.foxnews.com/sitemap.xml","Read the JSON-LD on the page (NewsMediaOrganization, WebSite, WebPage, CollectionPage). It is cleaner than the markup.","The site publishes 1 RSS or Atom feed. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example api.strike.fox/ctx/v1/tags. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":7,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.foxnews.com/sitemap.xml","https://www.foxnews.com/sitemap.xml?type=news"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["NewsMediaOrganization","WebSite","WebPage","CollectionPage","VideoObject"],"feeds":1,"feed_urls":["https://moxie.foxnews.com/google-publisher/latest.xml"],"api_endpoints_seen":["api.strike.fox/ctx/v1/tags","prod.pyxis.atp.fox/pyxis/submit","prod.idgraph.dt.fox/api/v2/item","prod1.fes.kts.fox/evs/fn"],"inner_page":{"url":"https://www.foxnews.com/about/company/media-relations/irena-briganti","status":200,"json_ld_types":["NewsMediaOrganization","WebSite","WebPage"],"skipped":null,"sitemap":"https://www.foxnews.com/sitemap.xml?type=pages"}},"performance":{"response_ms":51,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/foxnews.com.json"},{"schema":"ratradar.site/v1","host":"www.usatoday.com","name":"USA Today","category":"news","country":"United States","verdict":"guarded","hostility_score":26,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429.","Enumerate pages from the sitemap: https://www.usatoday.com/news-sitemap.xml","Read the JSON-LD on the page (Organization, WebSite, WebPage). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.usatoday.com/x1TGTCH4umKs/usatodaydemo/4/m%7Cd%7Cz%7Ck%7CFor%20Subscribers%20Bundle%7Cs%7Ce%7Cp%7Cy%7Cw%7Co%7Cr%7Cq%7Ct%7Ch/0/. Untested, check the terms before using them."],"risks":["robots.txt forbids 14 of 14 known AI crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["DataDome","Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":47,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","Claude-Web","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Bytespider","Amazonbot","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":14,"sitemap_urls":["https://www.usatoday.com/news-sitemap.xml","https://www.usatoday.com/web-sitemap-index.xml","https://www.usatoday.com/video-sitemap-index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["Organization","WebSite","WebPage"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.usatoday.com/x1TGTCH4umKs/usatodaydemo/4/m%7Cd%7Cz%7Ck%7CFor%20Subscribers%20Bundle%7Cs%7Ce%7Cp%7Cy%7Cw%7Co%7Cr%7Cq%7Ct%7Ch/0/","www.usatoday.com/pbd/cookie_sync","display.bidder.taboola.com/OpenRTB/TaboolaHB/auction","gum.criteo.com/sid/json","a.teads.tv/hb/bid-request"],"inner_page":{"url":"https://www.usatoday.com/story/sports/soccer/epl/2026/10/11/liverpool-lineup-vs-man-city-today/92168516007/","status":200,"json_ld_types":["NewsArticle"],"skipped":null,"sitemap":"https://www.usatoday.com/news-sitemap.xml"}},"performance":{"response_ms":113,"bytes":240342,"redirects":0,"cookies_set":9},"why_this_score":[{"label":"DataDome seen by the browser","points":20},{"label":"Cloudflare seen by the browser","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 14 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/usatoday.com.json"},{"schema":"ratradar.site/v1","host":"www.forbes.com","name":"Forbes","category":"news","country":"United States","verdict":"open","hostility_score":22,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Read the JSON-LD on the page (Organization). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.forbes.com/genai-flowapi/videos. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["DataDome","Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":"istio-envoy"},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":["Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.forbes.com/genai-flowapi/videos","www.forbes.com/simple-data/trending-picks","geolocation.forbes.com/json/","www.forbes.com/zephr/features"],"inner_page":null},"performance":{"response_ms":45,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"DataDome detected","points":20},{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/forbes.com.json"},{"schema":"ratradar.site/v1","host":"www.theverge.com","name":"The Verge","category":"news","country":"United States","verdict":"open","hostility_score":8,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.theverge.com/sitemaps/google_news","Read the JSON-LD on the page (NewsMediaOrganization, WebSite). It is cleaner than the markup.","The site publishes 1 RSS or Atom feed. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example www.theverge.com/api/kicker/today. Untested, check the terms before using them."],"risks":["robots.txt forbids 11 of 14 known AI crawlers (ChatGPT-User, ClaudeBot, Claude-Web, anthropic-ai, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly","Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":15,"crawl_delay_seconds":null,"ai_crawlers_blocked":["ChatGPT-User","ClaudeBot","Claude-Web","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Bytespider","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":11,"sitemap_urls":["https://www.theverge.com/sitemaps/google_news","https://www.theverge.com/sitemaps"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["NewsMediaOrganization","WebSite"],"feeds":1,"feed_urls":["https://www.theverge.com/rss/index.xml"],"api_endpoints_seen":["www.theverge.com/api/kicker/today","concertads-configs.vox-cdn.com/duet/verge/config.json"],"inner_page":{"url":"https://www.theverge.com/news/1008581/microsoft-365-family-premium-shared-ai-features-storage-changes","status":200,"json_ld_types":["NewsArticle","BreadcrumbList"],"skipped":null,"sitemap":"https://www.theverge.com/sitemaps/google_news"}},"performance":{"response_ms":57,"bytes":400000,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"Fastly detected","points":2},{"label":"Cloudflare seen by the browser","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 11 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/theverge.com.json"},{"schema":"ratradar.site/v1","host":"techcrunch.com","name":"TechCrunch","category":"news","country":"United States","verdict":"guarded","hostility_score":26,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429.","Enumerate pages from the sitemap: https://techcrunch.com/sitemap.xml","Read the JSON-LD on the page (CollectionPage, BreadcrumbList, WebSite, Organization). It is cleaner than the markup.","The site publishes 1 RSS or Atom feed. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example rp.liadm.com/j. Untested, check the terms before using them."],"risks":["robots.txt forbids 10 of 14 known AI crawlers (GPTBot, ChatGPT-User, ClaudeBot, Claude-Web, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare","PerimeterX"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":5,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","ClaudeBot","Claude-Web","anthropic-ai","CCBot","Google-Extended","Bytespider","Applebot-Extended","cohere-ai"],"ai_crawlers_blocked_count":10,"sitemap_urls":["https://techcrunch.com/sitemap.xml","https://techcrunch.com/news-sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["CollectionPage","BreadcrumbList","WebSite","Organization"],"feeds":1,"feed_urls":["https://techcrunch.com/feed/"],"api_endpoints_seen":["rp.liadm.com/j","clutchcourses.com/o20x238/bl5h7dnanyunr1v5f","ads.servenobid.com/adreq","hb.minutemedia-prebid.com/hb-mm-multi","htlb.casalemedia.com/openrtb/pbjs"],"inner_page":{"url":"https://techcrunch.com/2026/10/10/microsofts-satya-nadella-says-ai-models-need-an-emergency-brake/","status":200,"json_ld_types":["NewsArticle","WebPage","ImageObject","BreadcrumbList","WebSite","Organization","Person"],"skipped":null,"sitemap":"https://techcrunch.com/news-sitemap.xml"}},"performance":{"response_ms":69,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"PerimeterX seen by the browser","points":20}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 10 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/techcrunch.com.json"},{"schema":"ratradar.site/v1","host":"www.wired.com","name":"Wired","category":"news","country":"United States","verdict":"open","hostility_score":8,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.wired.com/sitemap.xml","The site publishes 1 RSS or Atom feed. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example www.wired.com/auth/refresh. Untested, check the terms before using them."],"risks":["robots.txt forbids 9 of 14 known AI crawlers (ClaudeBot, CCBot, PerplexityBot, Google-Extended, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare","CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":14,"crawl_delay_seconds":null,"ai_crawlers_blocked":["ClaudeBot","CCBot","PerplexityBot","Google-Extended","Bytespider","Amazonbot","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":9,"sitemap_urls":["https://www.wired.com/sitemap.xml","https://www.wired.com/tagpages-sitemap.xml","https://www.wired.com/contributors-sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":1,"feed_urls":["https://www.wired.com/feed/rss"],"api_endpoints_seen":["www.wired.com/auth/refresh","cdn.gladly.com/orgs/configs/chat/condenast.com-WIRED.json","yusui.conde.digital/v1","paywall.sophi.io/hosts/www.wired.com/decisions/me"],"inner_page":{"url":"https://www.wired.com/story/i-made-terrible-games-with-googles-ai-playground/","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.wired.com/feed/google-latest-news/sitemap-google-news"}},"performance":{"response_ms":143,"bytes":400000,"redirects":0,"cookies_set":5},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 9 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/wired.com.json"},{"schema":"ratradar.site/v1","host":"arstechnica.com","name":"Ars Technica","category":"news","country":"United States","verdict":"open","hostility_score":18,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://arstechnica.com/sitemap.xml","Read the JSON-LD on the page (WebSite, WebPage, Organization). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example planebasin.com/q1wh6sd28_2spn2l8f8ixm2puz0g8mek9nr4esn23k4. Untested, check the terms before using them."],"risks":["robots.txt forbids 11 of 14 known AI crawlers (ClaudeBot, Claude-Web, anthropic-ai, CCBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["AWS WAF","Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":30,"crawl_delay_seconds":null,"ai_crawlers_blocked":["ClaudeBot","Claude-Web","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Bytespider","Amazonbot","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":11,"sitemap_urls":["https://arstechnica.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebSite","WebPage","Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["planebasin.com/q1wh6sd28_2spn2l8f8ixm2puz0g8mek9nr4esn23k4","api.sail-personalize.com/v1/personalize/simple","www.googleadservices.com/pagead/conversion/816822159/"],"inner_page":{"url":"https://arstechnica.com/gadgets/1998/08/celeron128k/","status":200,"json_ld_types":["WebSite","WebPage","NewsArticle"],"skipped":null,"sitemap":"https://arstechnica.com/post-sitemap.xml"}},"performance":{"response_ms":519,"bytes":354453,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"AWS WAF detected","points":12},{"label":"Cloudflare seen by the browser","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 11 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/arstechnica.com.json"},{"schema":"ratradar.site/v1","host":"www.dailymail.co.uk","name":"Daily Mail","category":"news","country":"United Kingdom","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Enumerate pages from the sitemap: https://www.dailymail.co.uk/google-news-sitemap.xml","The site publishes 2 RSS or Atom feeds. Prefer them for new items."],"risks":["Hostile to automation. Keep volumes low and cache.","robots.txt forbids 14 of 14 known AI crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, ...). An agent that identifies as an AI crawler should not fetch this site.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":21,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","Claude-Web","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Bytespider","Amazonbot","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":14,"sitemap_urls":["https://www.dailymail.co.uk/google-news-sitemap.xml","https://www.dailymail.co.uk/sitemap-channels.xml","https://www.dailymail.co.uk/sitemap-columnist.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":2,"feed_urls":["https://www.dailymail.com/ushome/index.rss","https://www.dailymail.com/articles.rss"],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":644,"bytes":400000,"redirects":2,"cookies_set":4},"why_this_score":[{"label":"Akamai detected","points":10},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"strong","signals":[{"kind":"robots_txt","label":"robots.txt blocks 14 AI crawlers"},{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/dailymail.co.uk.json"},{"schema":"ratradar.site/v1","host":"www.thetimes.com","name":"The Times UK","category":"news","country":"United Kingdom","verdict":"hostile","hostility_score":50,"recommended_approach":"respect_robots_skip","steps":["robots.txt disallows every crawler. Do not scrape it. Look for an official API, a data license, or a partner feed.","Enumerate pages from the sitemap: https://www.thetimes.com/sitemaps/news","Read the JSON-LD on the page (CollectionPage). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example ads.thetimes.com/times_render.v2.raw.json. Untested, check the terms before using them."],"risks":["Scraping against an explicit robots.txt ban is the quickest way to a legal or blocking problem."],"can_scrape":{"plain_http":false,"headless_browser":true,"needs_proxies":false,"official_api_advised":true},"protections":{"vendors":["CloudFront","Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"CloudFront"},"robots":{"found":true,"disallow_all":true,"disallow_rules":1,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.thetimes.com/sitemaps/news","https://www.thetimes.com/sitemaps/articles/2026","https://www.thetimes.com/sitemaps/articles/2025"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["CollectionPage"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["ads.thetimes.com/times_render.v2.raw.json","ads.thetimes.com/dfp.json","newsuk-the-times.cdn.zephr.com/zephr/features","api.thetimes.co.uk/graphql"],"inner_page":null},"performance":{"response_ms":731,"bytes":400000,"redirects":0,"cookies_set":14},"why_this_score":[{"label":"CloudFront detected","points":2},{"label":"Cloudflare seen by the browser","points":6},{"label":"robots.txt disallows everything","points":10},{"label":"Raised to at least 50 because robots.txt disallows everything","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt bans every crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/thetimes.com.json"},{"schema":"ratradar.site/v1","host":"www.ft.com","name":"Financial Times","category":"news","country":"United Kingdom","verdict":"blocked","hostility_score":76,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies.","robots.txt asks for a crawl delay of 1 seconds.","Enumerate pages from the sitemap: https://www.ft.com/sitemaps/index.xml"],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans.","robots.txt forbids 10 of 14 known AI crawlers (ClaudeBot, Claude-Web, anthropic-ai, CCBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":14,"crawl_delay_seconds":1,"ai_crawlers_blocked":["ClaudeBot","Claude-Web","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Bytespider","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":10,"sitemap_urls":["https://www.ft.com/sitemaps/index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":68,"bytes":270345,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 10 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/ft.com.json"},{"schema":"ratradar.site/v1","host":"www.economist.com","name":"The Economist","category":"news","country":"United Kingdom","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies.","Enumerate pages from the sitemap: https://www.economist.com/sitemap.xml"],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans.","robots.txt forbids 11 of 14 known AI crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare","DataDome"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":11,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","Claude-Web","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Bytespider","Applebot-Extended"],"ai_crawlers_blocked_count":11,"sitemap_urls":["https://www.economist.com/sitemap.xml","https://www.economist.com/googlenews.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":50,"bytes":1137,"redirects":0,"cookies_set":4},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6},{"label":"DataDome detected","points":20}],"eu_ai_training":{"level":"strong","signals":[{"kind":"robots_txt","label":"robots.txt blocks 11 AI crawlers"},{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/economist.com.json"},{"schema":"ratradar.site/v1","host":"www.aljazeera.com","name":"Al Jazeera","category":"news","country":"United Arab Emirates","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.aljazeera.com/news-sitemap.xml","Read the JSON-LD on the page (CollectionPage, WebSite, NewsMediaOrganization). It is cleaner than the markup.","The site publishes 1 RSS or Atom feed. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example www.aljazeera.com/api/features. Untested, check the terms before using them."],"risks":["robots.txt forbids 8 of 14 known AI crawlers (GPTBot, ChatGPT-User, ClaudeBot, Claude-Web, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":15,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","ClaudeBot","Claude-Web","anthropic-ai","PerplexityBot","Bytespider","cohere-ai"],"ai_crawlers_blocked_count":8,"sitemap_urls":["https://www.aljazeera.com/news-sitemap.xml","https://www.aljazeera.com/sitemaps/article-archive.xml","https://www.aljazeera.com/sitemaps/article-new.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["CollectionPage","WebSite","NewsMediaOrganization"],"feeds":1,"feed_urls":["https://www.aljazeera.com/xml/rss/all.xml"],"api_endpoints_seen":["www.aljazeera.com/api/features","www.aljazeera.com/graphql"],"inner_page":{"url":"https://www.aljazeera.com/news/2026/10/11/ethiopia-eritrea-ramp-up-rhetoric-are-they-inching-towards-war","status":200,"json_ld_types":["NewsMediaOrganization","NewsArticle","WebPage","VideoObject","BreadcrumbList"],"skipped":null,"sitemap":"https://www.aljazeera.com/news-sitemap.xml"}},"performance":{"response_ms":83,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Cloudflare seen by the browser","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 8 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/aljazeera.com.json"},{"schema":"ratradar.site/v1","host":"www.dw.com","name":"DW","category":"news","country":"Germany","verdict":"guarded","hostility_score":25,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls.","Enumerate pages from the sitemap: https://www.dw.com/sitemap.xml","A browser load called JSON endpoints that look like data, for example www.dw.com/graph-api/en/default-nav. Untested, check the terms before using them."],"risks":["robots.txt forbids 14 of 14 known AI crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":18,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","Claude-Web","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Bytespider","Amazonbot","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":14,"sitemap_urls":["https://www.dw.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.dw.com/graph-api/en/default-nav","www.dw.com/graph-api/en/content/navigation/9097","www.dw.com/graph-api/en/road/translations","www.dw.com/graph-api/en/breaking-news"],"inner_page":{"url":"https://www.dw.com/de/der-s%C3%BCdsudan-auf-ungewissem-weg-zu-den-ersten-wahlen/a-79583983","status":200,"json_ld_types":["NewsArticle"],"skipped":null,"sitemap":"https://www.dw.com/de/news-sitemap.xml"}},"performance":{"response_ms":65,"bytes":12061,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15},{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 14 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/dw.com.json"},{"schema":"ratradar.site/v1","host":"www.france24.com","name":"France 24","category":"news","country":"France","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.france24.com/sitemaps/fr/index.xml","Read the JSON-LD on the page (BreadcrumbList, ItemList, WebSite). It is cleaner than the markup.","The site publishes 1 RSS or Atom feed. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example www.france24.com/en/_ws/urgent. Untested, check the terms before using them."],"risks":["robots.txt forbids 10 of 14 known AI crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","Claude-Web","anthropic-ai","CCBot","PerplexityBot","Google-Extended","cohere-ai"],"ai_crawlers_blocked_count":10,"sitemap_urls":["https://www.france24.com/sitemaps/fr/index.xml","https://www.france24.com/sitemaps/en/index.xml","https://www.france24.com/sitemaps/es/index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["BreadcrumbList","ItemList","WebSite"],"feeds":1,"feed_urls":["https://www.france24.com/en/rss"],"api_endpoints_seen":["www.france24.com/en/_ws/urgent"],"inner_page":{"url":"https://www.france24.com/fr/france/20261009-en-direct-france-mouvement-lyceen-acte-v-syndicats-lecornu","status":200,"json_ld_types":["NewsArticle","BreadcrumbList"],"skipped":null,"sitemap":"https://www.france24.com/sitemaps/fr/news.xml"}},"performance":{"response_ms":132,"bytes":400000,"redirects":1,"cookies_set":1},"why_this_score":[],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 10 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/france24.com.json"},{"schema":"ratradar.site/v1","host":"elpais.com","name":"El Pais","category":"news","country":"Spain","verdict":"open","hostility_score":20,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Read the JSON-LD on the page (BreadcrumbList, WebSite, WPHeader, WPFooter). It is cleaner than the markup.","The site publishes 1 RSS or Atom feed. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example ak-ads-ns.prisasd.com/slot/control/r/refreshAdGam.json. Untested, check the terms before using them."],"risks":["robots.txt forbids 5 of 14 known AI crawlers (ClaudeBot, CCBot, PerplexityBot, Bytespider, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["DataDome"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"openresty"},"robots":{"found":true,"disallow_all":false,"disallow_rules":16,"crawl_delay_seconds":null,"ai_crawlers_blocked":["ClaudeBot","CCBot","PerplexityBot","Bytespider","Amazonbot"],"ai_crawlers_blocked_count":5,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":["BreadcrumbList","WebSite","WPHeader","WPFooter","SiteNavigationElement","ItemList"],"feeds":1,"feed_urls":["https://feeds.elpais.com/mrss-s/pages/ep/site/elpais.com/section/us/portada"],"api_endpoints_seen":["ak-ads-ns.prisasd.com/slot/control/r/refreshAdGam.json","ak-ads-ns.prisasd.com/slot/elpais/data/lazyload.json","publicapi.elpais.com/sales/public/v1/entitlements","dd.elpais.com/js/","prisacom.tt.omtrdc.net/rest/v1/delivery"],"inner_page":null},"performance":{"response_ms":362,"bytes":399322,"redirects":1,"cookies_set":5},"why_this_score":[{"label":"DataDome detected","points":20}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 5 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/elpais.com.json"},{"schema":"ratradar.site/v1","host":"www.corriere.it","name":"Corriere","category":"news","country":"Italy","verdict":"open","hostility_score":8,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.corriere.it/feed-hp/homepage-restyle-2025.xml","Read the JSON-LD on the page (BreadcrumbList, ItemList, NewsMediaOrganization). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example mab.chartbeat.com/mab_strategy/headline_testing/get_strategy/. Untested, check the terms before using them."],"risks":["robots.txt forbids 11 of 14 known AI crawlers (GPTBot, ChatGPT-User, ClaudeBot, Claude-Web, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly","Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":80,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","ClaudeBot","Claude-Web","anthropic-ai","CCBot","Google-Extended","Bytespider","Amazonbot","Applebot-Extended","Meta-ExternalAgent"],"ai_crawlers_blocked_count":11,"sitemap_urls":["https://www.corriere.it/feed-hp/homepage-restyle-2025.xml","https://www.corriere.it/salute/sitemap-dizionario-corriere-salute.xml","https://www.corriere.it/dynamic-sitemap/sitemap-last-100/video/Corriere.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["BreadcrumbList","ItemList","NewsMediaOrganization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["mab.chartbeat.com/mab_strategy/headline_testing/get_strategy/","geoapi.corriere.it/internalgeo/countryCode2.js"],"inner_page":null},"performance":{"response_ms":510,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Fastly detected","points":2},{"label":"Cloudflare seen by the browser","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 11 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/corriere.it.json"},{"schema":"ratradar.site/v1","host":"www3.nhk.or.jp","name":"NHK","category":"news","country":"Japan","verdict":"guarded","hostility_score":45,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429."],"risks":["robots.txt forbids 12 of 14 known AI crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":1,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Bytespider","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":12,"sitemap_urls":["https://www3.nhk.or.jp/news/sitemap-news-index.xml","https://www3.nhk.or.jp/nhkworld/sitemap.xml"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":15040,"bytes":null,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"No answer to a plain request","points":40},{"label":"Slow response (15040 ms)","points":5}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 12 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":0,"headless_browser_status":0,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/www3.nhk.or.jp.json"},{"schema":"ratradar.site/v1","host":"timesofindia.indiatimes.com","name":"Times of India","category":"news","country":"India","verdict":"open","hostility_score":10,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://timesofindia.indiatimes.com/sitemap/today","A browser load called JSON endpoints that look like data, for example htlb.casalemedia.com/openrtb/pbjs. Untested, check the terms before using them."],"risks":["robots.txt forbids some AI crawlers: Meta-ExternalAgent."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":167,"crawl_delay_seconds":null,"ai_crawlers_blocked":["Meta-ExternalAgent"],"ai_crawlers_blocked_count":1,"sitemap_urls":["https://timesofindia.indiatimes.com/sitemap/today","https://timesofindia.indiatimes.com/sitemap/yesterday","https://timesofindia.indiatimes.com/staticsitemap/toi/news/sitemap-index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["htlb.casalemedia.com/openrtb/pbjs","web-banner.ads.aps.amazon-adsystem.com/e/dtb/bid","a.teads.tv/hb/bid-request","display.bidder.taboola.com/OpenRTB/TaboolaHB/auction","direct.adsrvr.org/bid/bidder/timesinternet","fastlane.rubiconproject.com/a/api/fastlane.json"],"inner_page":{"url":"https://timesofindia.indiatimes.com/entertainment/english/hollywood/news/quote-of-the-day-by-jamie-lee-curtis-it-is-not-ungrateful-to-want-more-youre-alive-/articleshow/131425207.cms","status":200,"json_ld_types":["WebPage","NewsArticle","ViewAction"],"skipped":null,"sitemap":"https://timesofindia.indiatimes.com/staticsitemap/toi/news/2026-May-1.xml"}},"performance":{"response_ms":342,"bytes":400002,"redirects":1,"cookies_set":0},"why_this_score":[{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 1 AI crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/timesofindia.indiatimes.com.json"},{"schema":"ratradar.site/v1","host":"www.scmp.com","name":"SCMP","category":"news","country":"Hong Kong","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","robots.txt asks for a crawl delay of 10 seconds.","Enumerate pages from the sitemap: https://www.scmp.com/sitemap/sitemap.xml","Read the JSON-LD on the page (Organization, NewsMediaOrganization, WebSite, ImageObject). It is cleaner than the markup.","The site publishes 1 RSS or Atom feed. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example apigw.scmp.com/content-delivery/v2. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":116,"crawl_delay_seconds":10,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.scmp.com/sitemap/sitemap.xml","https://www.scmp.com/sitemap/archives-0.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["Organization","NewsMediaOrganization","WebSite","ImageObject"],"feeds":1,"feed_urls":["https://www.scmp.com/rss/feed"],"api_endpoints_seen":["apigw.scmp.com/content-delivery/v2","bcp.crwdcntrl.net/6/map","apigw.scmp.com/navigator-info/v1","apigw.scmp.com/gloc/v1/"],"inner_page":{"url":"https://www.scmp.com/cooking/recipe/perfect-egg-fried-rice/article/3215340","status":200,"json_ld_types":["Recipe"],"skipped":null,"sitemap":"https://www.scmp.com/sitemap/recipes.xml"}},"performance":{"response_ms":62,"bytes":400000,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/scmp.com.json"},{"schema":"ratradar.site/v1","host":"www.abc.net.au","name":"ABC Australia","category":"news","country":"Australia","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.abc.net.au/sitemaps/sitemap-index.xml.gz","Read the JSON-LD on the page (WebSite, CollectionPage, NewsMediaOrganization). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.abc.net.au/news-web/api/loader/locationandweather. Untested, check the terms before using them."],"risks":["robots.txt forbids 11 of 14 known AI crawlers (GPTBot, ChatGPT-User, ClaudeBot, Claude-Web, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":24,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","ClaudeBot","Claude-Web","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Bytespider","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":11,"sitemap_urls":["https://www.abc.net.au/sitemaps/sitemap-index.xml.gz"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebSite","CollectionPage","NewsMediaOrganization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.abc.net.au/news-web/api/loader/locationandweather","recommendations.abc.net.au/api/v3/recommendation/VEJZdJIBjFK81s4i1MEc"],"inner_page":{"url":"https://www.abc.net.au/news/2026-07-27/abc-news/106964116","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.abc.net.au/sitemaps/sitemap-news-0.xml.gz"}},"performance":{"response_ms":232,"bytes":400000,"redirects":0,"cookies_set":3},"why_this_score":[],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 11 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/abc.net.au.json"},{"schema":"ratradar.site/v1","host":"www.cbc.ca","name":"CBC","category":"news","country":"Canada","verdict":"guarded","hostility_score":45,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":15042,"bytes":null,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"No answer to a plain request","points":40},{"label":"Slow response (15042 ms)","points":5}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":0,"headless_browser_status":0,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/cbc.ca.json"},{"schema":"ratradar.site/v1","host":"www.theglobeandmail.com","name":"Globe and Mail","category":"news","country":"Canada","verdict":"hostile","hostility_score":52,"recommended_approach":"real_browser","steps":["Use a rendering browser, slow requests, and rotating proxies.","Enumerate pages from the sitemap: https://www.theglobeandmail.com/sitemap.xml?outputType=xml","Read the JSON-LD on the page (NewsMediaOrganization). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.theglobeandmail.com/pf/api/v3/content/fetch/active-ad-campaigns. Untested, check the terms before using them."],"risks":["Hostile to automation. Keep volumes low and cache.","robots.txt forbids 13 of 14 known AI crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":false,"headless_browser":true,"needs_proxies":true,"official_api_advised":false},"protections":{"vendors":["Akamai","DataDome","CloudFront","PerimeterX"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"openresty"},"robots":{"found":true,"disallow_all":false,"disallow_rules":9,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","Claude-Web","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Bytespider","Amazonbot","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":13,"sitemap_urls":["https://www.theglobeandmail.com/sitemap.xml?outputType=xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["NewsMediaOrganization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.theglobeandmail.com/pf/api/v3/content/fetch/active-ad-campaigns"],"inner_page":{"url":"https://www.theglobeandmail.com/arc/outboundfeeds/news-sitemap/?outputType=xml","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.theglobeandmail.com/arc/outboundfeeds/news-sitemap-index/?outputType=xml"}},"performance":{"response_ms":281,"bytes":400000,"redirects":0,"cookies_set":4},"why_this_score":[{"label":"Akamai detected","points":10},{"label":"DataDome detected","points":20},{"label":"CloudFront detected","points":2},{"label":"PerimeterX seen by the browser","points":20}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 13 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/theglobeandmail.com.json"},{"schema":"ratradar.site/v1","host":"www.news24.com","name":"News24","category":"news","country":"South Africa","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Read the JSON-LD on the page (BreadcrumbList). It is cleaner than the markup."],"risks":["Hostile to automation. Keep volumes low and cache.","robots.txt forbids 14 of 14 known AI crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, ...). An agent that identifies as an AI crawler should not fetch this site.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":9,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","Claude-Web","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Bytespider","Amazonbot","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":14,"sitemap_urls":["https://www.news24.com/sitemap"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":["BreadcrumbList"],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":66,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 14 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/news24.com.json"},{"schema":"ratradar.site/v1","host":"www.folha.uol.com.br","name":"Folha","category":"news","country":"Brazil","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.folha.uol.com.br/sitemap.xml","Read the JSON-LD on the page (WebSite, WebPage, ImageObject). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example mab.chartbeat.com/mab_strategy/headline_testing/get_strategy/. Untested, check the terms before using them."],"risks":["robots.txt forbids 9 of 14 known AI crawlers (ClaudeBot, anthropic-ai, CCBot, PerplexityBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":5,"crawl_delay_seconds":null,"ai_crawlers_blocked":["ClaudeBot","anthropic-ai","CCBot","PerplexityBot","Bytespider","Amazonbot","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":9,"sitemap_urls":["https://www.folha.uol.com.br/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebSite","WebPage","ImageObject"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["mab.chartbeat.com/mab_strategy/headline_testing/get_strategy/","ib.adnxs.com/getuidj","upc.udr.uol.com.br/api/policyVersion","udr.uol.com.br/","web-banner.ads.aps.amazon-adsystem.com/e/dtb/bid"],"inner_page":{"url":"https://www1.folha.uol.com.br/blogs/tudo-mais-um-pouco/2026/10/como-cuidar-da-planta-fantasma.shtml","status":200,"json_ld_types":["CreativeWork","ReportageNewsArticle"],"skipped":null,"sitemap":"https://www.folha.uol.com.br/blogs/sitemap.xml"}},"performance":{"response_ms":1346,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 9 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/folha.uol.com.br.json"},{"schema":"ratradar.site/v1","host":"www.clarin.com","name":"Clarin","category":"news","country":"Argentina","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.clarin.com/sitemaps/index.xml","Read the JSON-LD on the page (Organization, ItemList, NewsMediaOrganization, Website). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example onetag-geo.s-onetag.com/. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":11,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.clarin.com/sitemaps/index.xml","https://www.clarin.com/sitemaps/index_services.xml","https://www.clarin.com/sitemaps/sitemap_google_news.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["Organization","ItemList","NewsMediaOrganization","Website","BreadcrumbList","SoftwareApplication"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["onetag-geo.s-onetag.com/","get.geojs.io/v1/ip/geo.json","contentaccessapi.zondadevops.com/api/content-access/process-page","ap.lijit.com/readerinfo/v2","www.clarin.com/api/pagegrowlerList/subscriptionBtn/false"],"inner_page":{"url":"https://www.clarin.com/videos/marina-castano-viuda-camilo-jose-cela_3_8yFrSfCRga.html","status":200,"json_ld_types":["BreadcrumbList","Organization","NewsMediaOrganization","NewsArticle"],"skipped":null,"sitemap":"https://www.clarin.com/sitemaps/sitemap_google_news.xml"}},"performance":{"response_ms":91,"bytes":400000,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/clarin.com.json"},{"schema":"ratradar.site/v1","host":"www.yahoo.com","name":"Yahoo","category":"news","country":"United States","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Read the JSON-LD on the page (WebSite, NewsMediaOrganization). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example s.yimg.com/eh/prebid-config/fp-us-desktop.json. Untested, check the terms before using them."],"risks":["robots.txt forbids 10 of 14 known AI crawlers (GPTBot, ChatGPT-User, ClaudeBot, Claude-Web, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"ATS"},"robots":{"found":true,"disallow_all":false,"disallow_rules":21,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","ClaudeBot","Claude-Web","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Bytespider","cohere-ai"],"ai_crawlers_blocked_count":10,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":["WebSite","NewsMediaOrganization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["s.yimg.com/eh/prebid-config/fp-us-desktop.json","s.yimg.com/eh/prebid-config/bp-fp.json","signal-service.pbs.yahoo.com/v1/signal/refresh","nexus-gateway-prod.media.yahoo.com/"],"inner_page":null},"performance":{"response_ms":550,"bytes":400000,"redirects":0,"cookies_set":3},"why_this_score":[],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 10 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/yahoo.com.json"},{"schema":"ratradar.site/v1","host":"www.bing.com","name":"Bing","category":"search","country":"United States","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","A browser load called JSON endpoints that look like data, for example www.bing.com/hp/api/model. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":95,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.bing.com/sitemap.xml"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.bing.com/hp/api/model","www.bing.com/hp/api/v1/trivia","www.bing.com/hp/api/v1/carousel","www.msn.com/pcs/api/widget/bingHomepage/widgetfeed","assets.msn.com/resolver/api/resolve/v3/config/","assets.msn.com/service/news/feed/pages/weblayout"],"inner_page":null},"performance":{"response_ms":114,"bytes":214704,"redirects":0,"cookies_set":12},"why_this_score":[{"label":"Cloudflare seen by the browser","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/bing.com.json"},{"schema":"ratradar.site/v1","host":"duckduckgo.com","name":"DuckDuckGo","category":"search","country":"United States","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://duckduckgo.com/sitemap.xml","A browser load called JSON endpoints that look like data, for example duckduckgo.com/country.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":5,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://duckduckgo.com/sitemap.xml","https://duckduckgo.com/duckduckgo-help-pages/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["duckduckgo.com/country.json"],"inner_page":{"url":"https://duckduckgo.com/assets/hiring/how_we_work.pdf","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://duckduckgo.com/sitemap.xml"}},"performance":{"response_ms":280,"bytes":353271,"redirects":0,"cookies_set":1},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/duckduckgo.com.json"},{"schema":"ratradar.site/v1","host":"search.brave.com","name":"Brave Search","category":"search","country":"United States","verdict":"hostile","hostility_score":60,"recommended_approach":"real_browser","steps":["Use a rendering browser, slow requests, and rotating proxies.","A browser load called JSON endpoints that look like data, for example search.brave.com/api/home. Untested, check the terms before using them."],"risks":["Hostile to automation. Keep volumes low and cache."],"can_scrape":{"plain_http":false,"headless_browser":true,"needs_proxies":true,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":10,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["search.brave.com/api/home","search.brave.com/api/feedback"],"inner_page":null},"performance":{"response_ms":71,"bytes":73846,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"HTTP 429 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"CloudFront detected","points":2},{"label":"A real browser got in though a plain request did not","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":429,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/search.brave.com.json"},{"schema":"ratradar.site/v1","host":"www.ecosia.org","name":"Ecosia","category":"search","country":"Germany","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","robots.txt asks for a crawl delay of 10 seconds."],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":19,"crawl_delay_seconds":10,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":190,"bytes":73354,"redirects":0,"cookies_set":5},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/ecosia.org.json"},{"schema":"ratradar.site/v1","host":"news.naver.com","name":"Naver News","category":"news","country":"South Korea","verdict":"hostile","hostility_score":50,"recommended_approach":"respect_robots_skip","steps":["robots.txt disallows every crawler. Do not scrape it. Look for an official API, a data license, or a partner feed.","A browser load called JSON endpoints that look like data, for example news.naver.com/channel/rightbar. Untested, check the terms before using them."],"risks":["Scraping against an explicit robots.txt ban is the quickest way to a legal or blocking problem.","robots.txt forbids 8 of 14 known AI crawlers (GPTBot, OAI-SearchBot, ClaudeBot, CCBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":false,"headless_browser":true,"needs_proxies":false,"official_api_advised":true},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"nfront"},"robots":{"found":true,"disallow_all":true,"disallow_rules":1,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","OAI-SearchBot","ClaudeBot","CCBot","PerplexityBot","Google-Extended","Applebot-Extended","Meta-ExternalAgent"],"ai_crawlers_blocked_count":8,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["news.naver.com/channel/rightbar"],"inner_page":null},"performance":{"response_ms":404,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"robots.txt disallows everything","points":10},{"label":"Raised to at least 50 because robots.txt disallows everything","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt bans every crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/news.naver.com.json"},{"schema":"ratradar.site/v1","host":"gitlab.com","name":"GitLab","category":"dev","country":"United States","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://gitlab.com/sitemap.xml","Read the JSON-LD on the page (WebSite, WebPage, Organization, Corporation). It is cleaner than the markup.","The site publishes 1 RSS or Atom feed. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example about.gitlab.com/_i18n/5ncwDb80/en-us/messages.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":86,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebSite","WebPage","Organization","Corporation"],"feeds":1,"feed_urls":["https://about.gitlab.com/releases.xml"],"api_endpoints_seen":["about.gitlab.com/_i18n/5ncwDb80/en-us/messages.json"],"inner_page":{"url":"https://gitlab.com/groups/gitlab-org/orbit/team/product/-/issues","status":200,"json_ld_types":["BreadcrumbList"],"skipped":null,"sitemap":"https://gitlab.com/sitemap.xml"}},"performance":{"response_ms":240,"bytes":219115,"redirects":1,"cookies_set":0},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/gitlab.com.json"},{"schema":"ratradar.site/v1","host":"bitbucket.org","name":"Bitbucket","category":"dev","country":"Australia","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Read the JSON-LD on the page (WebSite). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example api.atlassian.com/flags/api/v2/configurations. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"AtlassianEdge"},"robots":{"found":true,"disallow_all":false,"disallow_rules":38,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":["WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["api.atlassian.com/flags/api/v2/configurations","api.atlassian.com/flags/api/v2/frontend/clientSdkKey/bxp-magnolia_web","www.atlassian.com/endpoint/location","api.atlassian.com/flags/api/v2/frontend/experimentValues","cdn-mr.contentful.com/spaces/3s3v3nq72la0/environments/master/entries"],"inner_page":null},"performance":{"response_ms":501,"bytes":178447,"redirects":0,"cookies_set":3},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/bitbucket.org.json"},{"schema":"ratradar.site/v1","host":"www.npmjs.com","name":"npm","category":"dev","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":27,"bytes":5617,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/npmjs.com.json"},{"schema":"ratradar.site/v1","host":"pypi.org","name":"PyPI","category":"dev","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://pypi.org/sitemap.xml","The site publishes 2 RSS or Atom feeds. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example 2p66nmmycsj3.statuspage.io/api/v2/status.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":"gunicorn"},"robots":{"found":true,"disallow_all":false,"disallow_rules":12,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://pypi.org/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":2,"feed_urls":["https://pypi.org/rss/updates.xml","https://pypi.org/rss/packages.xml"],"api_endpoints_seen":["2p66nmmycsj3.statuspage.io/api/v2/status.json"],"inner_page":{"url":"https://pypi.org/project/102203594-topsis/","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://pypi.org/00.sitemap.xml"}},"performance":{"response_ms":24,"bytes":27991,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/pypi.org.json"},{"schema":"ratradar.site/v1","host":"hub.docker.com","name":"Docker Hub","category":"dev","country":"United States","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://d36jcksde1wxzq.cloudfront.net/sitemap/sitemap-index.xml","Read the JSON-LD on the page (WebSite). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example sessions-bugsnag.docker.com/. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":["x-ratelimit-ip","x-ratelimit-limit","x-ratelimit-remaining","x-ratelimit-reset"],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://d36jcksde1wxzq.cloudfront.net/sitemap/sitemap-index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["sessions-bugsnag.docker.com/","hub.docker.com/__manifest","hub.docker.com/v2/feature-flags/proxy"],"inner_page":{"url":"https://hub.docker.com/r/jmb12686/glusterfs-volume-plugin","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://d36jcksde1wxzq.cloudfront.net/sitemap/sitemap-001.xml.gz"}},"performance":{"response_ms":198,"bytes":400000,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/hub.docker.com.json"},{"schema":"ratradar.site/v1","host":"huggingface.co","name":"Hugging Face","category":"ai","country":"United States","verdict":"open","hostility_score":14,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://huggingface.co/sitemap.xml","The site publishes 1 RSS or Atom feed. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example m.stripe.com/6. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["AWS WAF","CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":["ratelimit","ratelimit-policy"],"cdn":"CloudFront","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://huggingface.co/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":1,"feed_urls":["https://huggingface.co/blog/feed.xml"],"api_endpoints_seen":["m.stripe.com/6"],"inner_page":{"url":"https://huggingface.co/datasets/NOAM212819/torat-melech","status":200,"json_ld_types":["sc:Dataset"],"skipped":null,"sitemap":"https://huggingface.co/sitemap-datasets.xml"}},"performance":{"response_ms":41,"bytes":173931,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"AWS WAF detected","points":12},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/huggingface.co.json"},{"schema":"ratradar.site/v1","host":"www.kaggle.com","name":"Kaggle","category":"dev","country":"United States","verdict":"open","hostility_score":15,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls.","A browser load called JSON endpoints that look like data, for example www.kaggle.com/api/i/users.UsersService/GetClientFeatureFlags. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":true,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.kaggle.com/api/i/users.UsersService/GetClientFeatureFlags","www.kaggle.com/api/i/users.UsersService/GetCurrentUser","www.kaggle.com/api/i/users.ModerationPublicService/ListViolationDescriptions","www.kaggle.com/api/i/users.HomePageService/GetLoggedOutHomepageData","www.kaggle.com/api/i/users.HatsService/GetUserHatsStatus"],"inner_page":null},"performance":{"response_ms":88,"bytes":5960,"redirects":0,"cookies_set":5},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/kaggle.com.json"},{"schema":"ratradar.site/v1","host":"arxiv.org","name":"arXiv","category":"edu","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","robots.txt asks for a crawl delay of 15 seconds.","A browser load called JSON endpoints that look like data, for example arxiv.org/institutional_banner. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":"Google Frontend"},"robots":{"found":true,"disallow_all":false,"disallow_rules":28,"crawl_delay_seconds":15,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["arxiv.org/institutional_banner"],"inner_page":null},"performance":{"response_ms":26,"bytes":38033,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/arxiv.org.json"},{"schema":"ratradar.site/v1","host":"developer.mozilla.org","name":"MDN","category":"dev","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://developer.mozilla.org/sitemap.xml","The site publishes 1 RSS or Atom feed. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example developer.mozilla.org/api/v1/whoami. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":"Google Frontend"},"robots":{"found":true,"disallow_all":false,"disallow_rules":3,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://developer.mozilla.org/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":1,"feed_urls":["https://developer.mozilla.org/en-US/blog/rss.xml"],"api_endpoints_seen":["developer.mozilla.org/api/v1/whoami","developer.mozilla.org/pong/get"],"inner_page":{"url":"https://developer.mozilla.org/es/docs/Web/API/Event/Event","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://developer.mozilla.org/sitemaps/es/sitemap.xml.gz"}},"performance":{"response_ms":89,"bytes":121671,"redirects":1,"cookies_set":0},"why_this_score":[{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/developer.mozilla.org.json"},{"schema":"ratradar.site/v1","host":"stackexchange.com","name":"Stack Exchange","category":"dev","country":"United States","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","The site publishes 1 RSS or Atom feed. Prefer them for new items."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":1,"feed_urls":["https://stackexchange.com/feeds/questions"],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":135,"bytes":107962,"redirects":0,"cookies_set":4},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/stackexchange.com.json"},{"schema":"ratradar.site/v1","host":"vercel.com","name":"Vercel","category":"dev","country":"United States","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://vercel.com/sitemap.xml","Read the JSON-LD on the page (Organization, Service, SoftwareApplication). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example vercel.com/.well-known/vercel/microfrontends/client-config. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Vercel","server":"Vercel"},"robots":{"found":true,"disallow_all":false,"disallow_rules":10,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://vercel.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["Organization","Service","SoftwareApplication"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["vercel.com/.well-known/vercel/microfrontends/client-config","vercel.com/api/client-only-flags/show-platform-status-indicator"],"inner_page":{"url":"https://vercel.com/products/observability","status":200,"json_ld_types":["SoftwareApplication"],"skipped":null,"sitemap":"https://vercel.com/sitemap.xml"}},"performance":{"response_ms":137,"bytes":400000,"redirects":0,"cookies_set":3},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/vercel.com.json"},{"schema":"ratradar.site/v1","host":"www.netlify.com","name":"Netlify","category":"dev","country":"United States","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.netlify.com/sitemap-index.xml","Read the JSON-LD on the page (Organization, WebSite, WebPage, BreadcrumbList). It is cleaner than the markup.","The site publishes 3 RSS or Atom feeds. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example www.netlify.com/pagefind/pagefind-entry.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Netlify","server":"Netlify"},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.netlify.com/sitemap-index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["Organization","WebSite","WebPage","BreadcrumbList","SoftwareApplication"],"feeds":3,"feed_urls":["https://www.netlify.com/feed.xml","https://www.netlify.com/changelog/feed.xml","https://www.netlify.com/knowledge-base/feed.xml"],"api_endpoints_seen":["www.netlify.com/pagefind/pagefind-entry.json"],"inner_page":{"url":"https://www.netlify.com/blog/news/page/1/","status":200,"json_ld_types":["Organization","WebSite","WebPage","BreadcrumbList","SoftwareApplication"],"skipped":null,"sitemap":"https://www.netlify.com/sitemap-0.xml"}},"performance":{"response_ms":342,"bytes":400000,"redirects":0,"cookies_set":3},"why_this_score":[{"label":"Cloudflare seen by the browser","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/netlify.com.json"},{"schema":"ratradar.site/v1","host":"www.heroku.com","name":"Heroku","category":"dev","country":"United States","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.heroku.com/sitemap_index.xml","Read the JSON-LD on the page (WebPage, WebSite, Corporation, SearchAction). It is cleaner than the markup.","The site publishes 1 RSS or Atom feed. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example api.parsely.com/v2/profile. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":5,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.heroku.com/sitemap_index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["WebPage","WebSite","Corporation","SearchAction"],"feeds":1,"feed_urls":["https://www.heroku.com/feed/"],"api_endpoints_seen":["api.parsely.com/v2/profile","www.heroku.com/wp-admin/admin-ajax.php"],"inner_page":{"url":"https://www.heroku.com/events/2025/greenio-new-york/","status":200,"json_ld_types":["Event","WebSite","Corporation","BreadcrumbList","SearchAction"],"skipped":null,"sitemap":"https://www.heroku.com/heroku_event-sitemap.xml"}},"performance":{"response_ms":84,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/heroku.com.json"},{"schema":"ratradar.site/v1","host":"www.digitalocean.com","name":"DigitalOcean","category":"dev","country":"United States","verdict":"open","hostility_score":8,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.digitalocean.com/sitemap.xml","A browser load called JSON endpoints that look like data, for example www.digitalocean.com/api/static-content/v1/global_contents/top_hat. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare","CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":4,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.digitalocean.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.digitalocean.com/api/static-content/v1/global_contents/top_hat"],"inner_page":{"url":"https://www.digitalocean.com/products/droplets/cpu-optimized","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.digitalocean.com/sitemaps/products-0.xml.gz"}},"performance":{"response_ms":55,"bytes":291795,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/digitalocean.com.json"},{"schema":"ratradar.site/v1","host":"aws.amazon.com","name":"AWS","category":"dev","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://aws.amazon.com/sitemaps/index/","Read the JSON-LD on the page (ItemList). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example vs.aws.amazon.com/token. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"Server"},"robots":{"found":true,"disallow_all":false,"disallow_rules":266,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://aws.amazon.com/sitemaps/index/","https://d36a099zv34xie.cloudfront.net/psf-sitemap.xml","https://aws.amazon.com/marketplace/sitemap"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["ItemList"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["vs.aws.amazon.com/token","vs.aws.amazon.com/vs/ad-token","infra-api.us-east-1.prod.mrc-sunrise.marketing.aws.dev/custom-metric","edge.adobedc.net/ee/or2/v1/interact"],"inner_page":{"url":"https://aws.amazon.com/jp/about-aws/events/2016/atoz-osaka/register/","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://aws.amazon.com/sitemaps/sitemap_japan/"}},"performance":{"response_ms":112,"bytes":400000,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/aws.amazon.com.json"},{"schema":"ratradar.site/v1","host":"azure.microsoft.com","name":"Azure","category":"dev","country":"United States","verdict":"open","hostility_score":10,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://azure.microsoft.com/robotsitemap-en-us/","A browser load called JSON endpoints that look like data, for example uhf-exp-fd-gbcrdgggfbggh0g3.b02.azurefd.net/api/v1/tas. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":25,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://azure.microsoft.com/robotsitemap-en-us/","https://azure.microsoft.com/robotsitemap-cs-cz/","https://azure.microsoft.com/robotsitemap-da-dk/"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["uhf-exp-fd-gbcrdgggfbggh0g3.b02.azurefd.net/api/v1/tas","edge.adobedc.net/ee/v1/identity/acquire","azure.microsoft.com/msonecloudapi/customerstory/searchstories","www.microsoft.com/fe/api/features/variants"],"inner_page":{"url":"https://azure.microsoft.com/en-us/free/apps/search","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://azure.microsoft.com/robotsitemap-en-us-1/"}},"performance":{"response_ms":1729,"bytes":400000,"redirects":1,"cookies_set":1},"why_this_score":[{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/azure.microsoft.com.json"},{"schema":"ratradar.site/v1","host":"cloud.google.com","name":"Google Cloud","category":"dev","country":"United States","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://cloud.google.com/sitemap.xml","A browser load called JSON endpoints that look like data, for example www.gstatic.com/glue/cookienotificationbar/config/2a.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Google","server":"ESF"},"robots":{"found":true,"disallow_all":false,"disallow_rules":24,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://cloud.google.com/sitemap.xml","https://cloud.google.com/sitemapindex.xml","https://cloud.google.com/cgc/sitemapindex.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.gstatic.com/glue/cookienotificationbar/config/2a.json","cloud.google.com/__/pingz"],"inner_page":null},"performance":{"response_ms":385,"bytes":400000,"redirects":0,"cookies_set":1},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/cloud.google.com.json"},{"schema":"ratradar.site/v1","host":"www.perplexity.ai","name":"Perplexity","category":"ai","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies.","Enumerate pages from the sitemap: https://www.perplexity.ai/sitemap.xml"],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":6,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.perplexity.ai/sitemap.xml","https://www.perplexity.ai/help-center/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":112,"bytes":5578,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/perplexity.ai.json"},{"schema":"ratradar.site/v1","host":"mistral.ai","name":"Mistral","category":"ai","country":"France","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://mistral.ai/sitemap-index.xml","Read the JSON-LD on the page (Organization, WebSite). It is cleaner than the markup.","The site publishes 1 RSS or Atom feed. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example client.axept.io/projectconfigurations/brands/6a0b8527e6c59986c40d26de.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://mistral.ai/sitemap-index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["Organization","WebSite"],"feeds":1,"feed_urls":["https://mistral.ai/news/rss"],"api_endpoints_seen":["client.axept.io/projectconfigurations/brands/6a0b8527e6c59986c40d26de.json"],"inner_page":{"url":"https://mistral.ai/fr/news/2024-ft-hackathon/","status":200,"json_ld_types":["BlogPosting"],"skipped":null,"sitemap":"https://mistral.ai/sitemap-0.xml"}},"performance":{"response_ms":120,"bytes":334838,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/mistral.ai.json"},{"schema":"ratradar.site/v1","host":"cohere.com","name":"Cohere","category":"ai","country":"Canada","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://cohere.com/sitemap.xml","Read the JSON-LD on the page (Organization). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example cdn.sanity.io/files/rjtqmwfu/web3-prod/5f6ca0c21d6fe336ebc09c3def5eb29a5041e8d4.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Vercel","server":"Vercel"},"robots":{"found":true,"disallow_all":false,"disallow_rules":11,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://cohere.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["cdn.sanity.io/files/rjtqmwfu/web3-prod/5f6ca0c21d6fe336ebc09c3def5eb29a5041e8d4.json","rjtqmwfu.api.sanity.io/v2025-10-29/data/query/web3-prod"],"inner_page":{"url":"https://cohere.com/blog/iso-42001-and-iso-27001-certifications","status":200,"json_ld_types":["Organization","BlogPosting"],"skipped":null,"sitemap":"https://cohere.com/sitemap.xml"}},"performance":{"response_ms":109,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/cohere.com.json"},{"schema":"ratradar.site/v1","host":"www.midjourney.com","name":"Midjourney","category":"ai","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":42,"bytes":5643,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/midjourney.com.json"},{"schema":"ratradar.site/v1","host":"replicate.com","name":"Replicate","category":"ai","country":"United States","verdict":"open","hostility_score":8,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://replicate.com/sitemap.xml","A browser load called JSON endpoints that look like data, for example replicate.com/__manifest. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare","CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://replicate.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["replicate.com/__manifest","www.cloudflarestatus.com/api/v2/summary.json"],"inner_page":{"url":"https://replicate.com/newwork062026-lab/stocksurgemodel","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://replicate.com/sitemap-models.xml"}},"performance":{"response_ms":539,"bytes":353924,"redirects":0,"cookies_set":3},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/replicate.com.json"},{"schema":"ratradar.site/v1","host":"archive.org","name":"Archive.org","category":"dev","country":"United States","verdict":"open","hostility_score":15,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls.","Enumerate pages from the sitemap: https://archive.org/sitemap/sitemap.xml","A browser load called JSON endpoints that look like data, for example sink.archive.org/api/2/envelope/. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"nginx/1.31.3"},"robots":{"found":true,"disallow_all":false,"disallow_rules":2,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://archive.org/sitemap/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["sink.archive.org/api/2/envelope/","archive.org/services/offshoot/home-page/mediacounts.php","archive.org/services/donations/banner.php","archive.org/services/offshoot/home-page/announcements.php","archive.org/services/offshoot/config.php"],"inner_page":{"url":"https://archive.org/details/planning-board-september-28-2016_202511","status":200,"json_ld_types":["BreadcrumbList"],"skipped":null,"sitemap":"https://archive.org/sitemap/sitemap_00000.xml.gz"}},"performance":{"response_ms":96,"bytes":1872,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/archive.org.json"},{"schema":"ratradar.site/v1","host":"www.wikidata.org","name":"Wikidata","category":"dev","country":"Germany","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","The site publishes 1 RSS or Atom feed. Prefer them for new items."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"mw-web.codfw.main-6b979c7866-9b2cm"},"robots":{"found":true,"disallow_all":false,"disallow_rules":212,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.wikidata.org/w/rest.php/site/v1/sitemap/0"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":1,"feed_urls":["https://www.wikidata.org/w/index.php?title=Special:RecentChanges&feed=atom"],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":156,"bytes":213141,"redirects":1,"cookies_set":5},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/wikidata.org.json"},{"schema":"ratradar.site/v1","host":"www.openstreetmap.org","name":"OpenStreetMap","category":"dev","country":"United Kingdom","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":"Apache"},"robots":{"found":true,"disallow_all":false,"disallow_rules":21,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":837,"bytes":32317,"redirects":0,"cookies_set":3},"why_this_score":[{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/openstreetmap.org.json"},{"schema":"ratradar.site/v1","host":"www.wolframalpha.com","name":"Wolfram Alpha","category":"dev","country":"United States","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","A browser load called JSON endpoints that look like data, for example www.wolframalpha.com/n/v1/api/randomizer. Untested, check the terms before using them."],"risks":["robots.txt forbids 3 of 14 known AI crawlers (GPTBot, ChatGPT-User, Google-Extended). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":1,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","Google-Extended"],"ai_crawlers_blocked_count":3,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.wolframalpha.com/n/v1/api/randomizer"],"inner_page":null},"performance":{"response_ms":468,"bytes":181424,"redirects":0,"cookies_set":0},"why_this_score":[],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 3 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/wolframalpha.com.json"},{"schema":"ratradar.site/v1","host":"www.notion.so","name":"Notion","category":"saas","country":"United States","verdict":"open","hostility_score":8,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.notion.com/sitemap.xml","A browser load called JSON endpoints that look like data, for example www.notion.com/front-api/releases/latest-byline. Untested, check the terms before using them."],"risks":["robots.txt forbids some AI crawlers: Amazonbot."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare","CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":11,"crawl_delay_seconds":null,"ai_crawlers_blocked":["Amazonbot"],"ai_crawlers_blocked_count":1,"sitemap_urls":["https://www.notion.com/sitemap.xml","https://www.notion.com/blog/sitemap.xml","https://www.notion.com/customers/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.notion.com/front-api/releases/latest-byline","www.notion.com/front-api/notifications/en-US","exp.notion.com/v1/rgstr","exp.notion.com/v1/initialize"],"inner_page":null},"performance":{"response_ms":166,"bytes":208900,"redirects":1,"cookies_set":3},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 1 AI crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/notion.so.json"},{"schema":"ratradar.site/v1","host":"www.figma.com","name":"Figma","category":"saas","country":"United States","verdict":"open","hostility_score":8,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.figma.com/sitemap.xml","Read the JSON-LD on the page (Organization). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.figma.com/api/geoip. Untested, check the terms before using them."],"risks":["robots.txt forbids 6 of 14 known AI crawlers (GPTBot, ClaudeBot, CCBot, Google-Extended, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront","Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":17,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ClaudeBot","CCBot","Google-Extended","Bytespider","cohere-ai"],"ai_crawlers_blocked_count":6,"sitemap_urls":[]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.figma.com/api/geoip"],"inner_page":{"url":"https://www.figma.com/blog/news/","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.figma.com/sitemaps/index.xml"}},"performance":{"response_ms":218,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"CloudFront detected","points":2},{"label":"Cloudflare seen by the browser","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 6 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/figma.com.json"},{"schema":"ratradar.site/v1","host":"slack.com","name":"Slack","category":"saas","country":"United States","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://slack.com/sitemap.xml","A browser load called JSON endpoints that look like data, for example api.company-target.com/api/v3/ip.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"Apache"},"robots":{"found":true,"disallow_all":false,"disallow_rules":15,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://slack.com/sitemap.xml","https://slack.com/sitemap_marketplace.xml","https://slack.com/sitemaps/blog/news/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["api.company-target.com/api/v3/ip.json"],"inner_page":null},"performance":{"response_ms":515,"bytes":241371,"redirects":0,"cookies_set":4},"why_this_score":[{"label":"Cloudflare seen by the browser","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/slack.com.json"},{"schema":"ratradar.site/v1","host":"zoom.us","name":"Zoom","category":"saas","country":"United States","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://zoom.us/sitemap.xml","Read the JSON-LD on the page (Corporation, WebSite, WebPage). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example ixfd2-api.bc0a.com/api/ixf/1.0.0/get_capsule/f00000000314502/02136793597. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":28,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://zoom.us/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["Corporation","WebSite","WebPage"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["ixfd2-api.bc0a.com/api/ixf/1.0.0/get_capsule/f00000000314502/02136793597","zoom.us/buy/cart/get"],"inner_page":{"url":"https://explore.zoom.us/de/products/zoom-phone/features/virtual-phone-number/","status":200,"json_ld_types":["Corporation"],"skipped":null,"sitemap":"https://zoom.us/sitemap.xml"}},"performance":{"response_ms":101,"bytes":319976,"redirects":1,"cookies_set":1},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/zoom.us.json"},{"schema":"ratradar.site/v1","host":"www.dropbox.com","name":"Dropbox","category":"saas","country":"United States","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.dropbox.com/sitemapindex.xml","A browser load called JSON endpoints that look like data, for example edge.cofra.me/api/v1/frames. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"envoy"},"robots":{"found":true,"disallow_all":false,"disallow_rules":23,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.dropbox.com/sitemapindex.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["edge.cofra.me/api/v1/frames","www.dropbox.com/2/campaigns_toolkit/get_best_campaigns_for_user"],"inner_page":{"url":"https://www.dropbox.com/business/trust/compliance/certifications-compliance","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.dropbox.com/sitemap-warp.xml"}},"performance":{"response_ms":698,"bytes":400000,"redirects":0,"cookies_set":5},"why_this_score":[{"label":"Cloudflare seen by the browser","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/dropbox.com.json"},{"schema":"ratradar.site/v1","host":"www.atlassian.com","name":"Atlassian","category":"dev","country":"Australia","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.atlassian.com/sitemap.xml","A browser load called JSON endpoints that look like data, for example api.atlassian.com/flags/api/v2/configurations. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"AtlassianEdge"},"robots":{"found":true,"disallow_all":false,"disallow_rules":10,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.atlassian.com/sitemap.xml","https://www.atlassian.com/blog/post-sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["api.atlassian.com/flags/api/v2/configurations","www.atlassian.com/gateway/api/ais/available-products","www.atlassian.com/gateway/api/me","www.atlassian.com/endpoint/location","www.atlassian.com/gateway/api/graphql"],"inner_page":{"url":"https://www.atlassian.com/blog/archives/jira_saves_shine_technologies_over_100000_case_study","status":200,"json_ld_types":["WebPage","BreadcrumbList","WebSite"],"skipped":null,"sitemap":"https://www.atlassian.com/blog/post-sitemap.xml"}},"performance":{"response_ms":727,"bytes":400000,"redirects":0,"cookies_set":5},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/atlassian.com.json"},{"schema":"ratradar.site/v1","host":"finance.yahoo.com","name":"Yahoo Finance","category":"finance","country":"United States","verdict":"open","hostility_score":12,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://finance.yahoo.com/sitemap_en-us_quotes_index.xml","Read the JSON-LD on the page (WebSite). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example s.yimg.com/eh/prebid-config/finance-us-desktop.json. Untested, check the terms before using them."],"risks":["robots.txt forbids 11 of 14 known AI crawlers (GPTBot, ChatGPT-User, ClaudeBot, Claude-Web, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai","CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"ATS"},"robots":{"found":true,"disallow_all":false,"disallow_rules":32,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","ClaudeBot","Claude-Web","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Bytespider","Amazonbot","cohere-ai"],"ai_crawlers_blocked_count":11,"sitemap_urls":["https://finance.yahoo.com/sitemap_en-us_quotes_index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["s.yimg.com/eh/prebid-config/finance-us-desktop.json","s.yimg.com/eh/prebid-config/bp-finance.json","loc.yahoo.com/geo/v1/user/location/current","signal-service.pbs.yahoo.com/v1/signal/refresh"],"inner_page":{"url":"https://finance.yahoo.com/quote/ZWT=F/summary/","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://finance.yahoo.com/sitemaps/quotes/quote-0.xml"}},"performance":{"response_ms":382,"bytes":400000,"redirects":0,"cookies_set":3},"why_this_score":[{"label":"Akamai detected","points":10},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 11 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/finance.yahoo.com.json"},{"schema":"ratradar.site/v1","host":"www.marketwatch.com","name":"MarketWatch","category":"finance","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"respect_robots_skip","steps":["robots.txt disallows every crawler. Do not scrape it. Look for an official API, a data license, or a partner feed.","Enumerate pages from the sitemap: https://www.marketwatch.com/sitemaps/web/mw/en/sitemap_mw_en_index.xml"],"risks":["Scraping against an explicit robots.txt ban is the quickest way to a legal or blocking problem."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":false,"official_api_advised":true},"protections":{"vendors":["DataDome","CloudFront"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"CloudFront"},"robots":{"found":true,"disallow_all":true,"disallow_rules":1,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.marketwatch.com/sitemaps/web/mw/en/sitemap_mw_en_index.xml","https://www.marketwatch.com/sitemaps/web/mw-video/en/sitemap_mw-video_en_index.xml","https://www.marketwatch.com/mw_news_sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":62,"bytes":775,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 401 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"DataDome detected","points":20},{"label":"CloudFront detected","points":2},{"label":"robots.txt disallows everything","points":10},{"label":"Raised to at least 50 because robots.txt disallows everything","points":0}],"eu_ai_training":{"level":"strong","signals":[{"kind":"robots_txt","label":"robots.txt bans every crawler"},{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":401,"headless_browser_status":401,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/marketwatch.com.json"},{"schema":"ratradar.site/v1","host":"www.investing.com","name":"Investing.com","category":"finance","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Enumerate pages from the sitemap: https://www.investing.com/sitemap_index.xml"],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":33,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.investing.com/sitemap_index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.investing.com/news/earnings/vinfast-faces-earnings-test-as-asia-expansion-meets-profitability-push-93CH-4942684","status":403,"json_ld_types":[],"skipped":null,"sitemap":"https://www.investing.com/news_sitemap.xml"}},"performance":{"response_ms":290,"bytes":400000,"redirects":0,"cookies_set":10},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/investing.com.json"},{"schema":"ratradar.site/v1","host":"coinmarketcap.com","name":"CoinMarketCap","category":"finance","country":"Singapore","verdict":"open","hostility_score":20,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://s3.coinmarketcap.com/sitemap/sitemap.xml","Read the JSON-LD on the page (Organization, Table). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example coinmarketcap.com/locales/en/sheet_CMC%20AI.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["AWS WAF","CloudFront","Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":12,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://s3.coinmarketcap.com/sitemap/sitemap.xml","https://s3.coinmarketcap.com/sitemap/sitemap-CDP.xml","https://s3.coinmarketcap.com/sitemap/sitemap-CDPHistoricalData.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["Organization","Table"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["coinmarketcap.com/locales/en/sheet_CMC%20AI.json","coinmarketcap.com/locales/en/dexer-trade.json","api.coinmarketcap.com/data-api/v3/unified-trending/top-boost/listing","api.coinmarketcap.com/data-api/v3/unified-trending/listing","api.coinmarketcap.com/geo-info"],"inner_page":{"url":"https://coinmarketcap.com/top-stories/69af60514cb6434b85f74c8a/","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://s3.coinmarketcap.com/sitemap/sitemap-TopStoriesArticle.xml"}},"performance":{"response_ms":37,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"AWS WAF detected","points":12},{"label":"CloudFront detected","points":2},{"label":"Cloudflare seen by the browser","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/coinmarketcap.com.json"},{"schema":"ratradar.site/v1","host":"www.coingecko.com","name":"CoinGecko","category":"finance","country":"Singapore","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Enumerate pages from the sitemap: https://www.coingecko.com/sitemap.xml","Read the JSON-LD on the page (Organization). It is cleaner than the markup."],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare","PerimeterX"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":132,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.coingecko.com/sitemap.xml","https://www.coingecko.com/sitemaps/currency/sitemap.xml","https://www.coingecko.com/sitemaps/categories/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.coingecko.com/research/publications/2020-coingecko-yearly-crypto-report","status":200,"json_ld_types":["Article"],"skipped":null,"sitemap":"https://www.coingecko.com/sitemaps/posts/sitemap1.xml"}},"performance":{"response_ms":113,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"PerimeterX detected","points":20},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/coingecko.com.json"},{"schema":"ratradar.site/v1","host":"www.coinbase.com","name":"Coinbase","category":"finance","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Enumerate pages from the sitemap: https://www.coinbase.com/sitemap-converter-index.xml","Read the JSON-LD on the page (Organization). It is cleaner than the markup."],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare","CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":22,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.coinbase.com/sitemap-converter-index.xml","https://www.coinbase.com/sitemap-derivatives-index.xml","https://www.coinbase.com/sitemap-prediction-markets-index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.coinbase.com/converter/ip/a","status":403,"json_ld_types":[],"skipped":null,"sitemap":"https://www.coinbase.com/sitemap-converter-asset-a.xml"}},"performance":{"response_ms":206,"bytes":400000,"redirects":0,"cookies_set":3},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"CloudFront detected","points":2},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/coinbase.com.json"},{"schema":"ratradar.site/v1","host":"www.binance.com","name":"Binance","category":"finance","country":"Singapore","verdict":"hostile","hostility_score":59,"recommended_approach":"real_browser","steps":["Use a rendering browser, slow requests, and rotating proxies.","Enumerate pages from the sitemap: https://www.binance.com/sitemap_output/domain=www.binance.com/sitemap_Trade_index.xml","A browser load called JSON endpoints that look like data, for example www.binance.com/bapi/accounts/v1/public/authcenter/auth. Untested, check the terms before using them."],"risks":["Hostile to automation. Keep volumes low and cache."],"can_scrape":{"plain_http":false,"headless_browser":true,"needs_proxies":true,"official_api_advised":false},"protections":{"vendors":["AWS WAF","CloudFront"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"CloudFront"},"robots":{"found":true,"disallow_all":false,"disallow_rules":20,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.binance.com/sitemap_output/domain=www.binance.com/sitemap_Trade_index.xml","https://www.binance.com/sitemap_output/domain=www.binance.com/sitemap_Blog_index.xml","https://www.binance.com/sitemap_output/domain=www.binance.com/sitemap_Nft_index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.binance.com/bapi/accounts/v1/public/authcenter/auth","bin.bnbstatic.com/api/i18n/-/web/cms/en/binance-cookie-manager","www.binance.com/bapi/asset/v1/public/asset-service/product/currency"],"inner_page":{"url":"https://www.binance.com/en/trade/%E5%B8%81%E5%AE%89%E4%BA%BA%E7%94%9F_U","status":202,"json_ld_types":[],"skipped":null,"sitemap":"https://www.binance.com/sitemap_output/domain=www.binance.com/sitemap_Trade_en_0.xml"}},"performance":{"response_ms":48,"bytes":2105,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"A challenge page appeared","points":45},{"label":"AWS WAF detected","points":12},{"label":"CloudFront detected","points":2},{"label":"A real browser got in though a plain request did not","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":202,"headless_browser_status":202,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/binance.com.json"},{"schema":"ratradar.site/v1","host":"robinhood.com","name":"Robinhood","category":"finance","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://robinhood.com/sitemap.xml","A browser load called JSON endpoints that look like data, for example api.usercentrics.eu/ruleSet/M6Jkl5BTgFMHMJ.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"envoy"},"robots":{"found":true,"disallow_all":false,"disallow_rules":7,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://robinhood.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["api.usercentrics.eu/ruleSet/M6Jkl5BTgFMHMJ.json","bonfire.robinhood.com/region","api.usercentrics.eu/settings/XpXabaxP5dY-KV/latest/languages.json"],"inner_page":{"url":"https://docs.robinhood.com/crypto/connect/","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://docs.robinhood.com/sitemap.xml"}},"performance":{"response_ms":463,"bytes":266984,"redirects":1,"cookies_set":0},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/robinhood.com.json"},{"schema":"ratradar.site/v1","host":"www.paypal.com","name":"PayPal","category":"finance","country":"United States","verdict":"open","hostility_score":22,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.paypal.com/paypal-sitemap-index.xml","Read the JSON-LD on the page (Organization, Corporation, WebSite, WebPage). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.paypal.com/myaccount/privacy/cookieprefs/getBanner. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly","DataDome"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":91,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.paypal.com/paypal-sitemap-index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["Organization","Corporation","WebSite","WebPage","PaymentCard","LoanOrCredit","CreditCard","FinancialProduct"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.paypal.com/myaccount/privacy/cookieprefs/getBanner","browser-intake-us5-datadoghq.com/api/v2/rum","www.paypal.com/myaccount/privacy/cookieprefs/cookies","www.paypal.com/smartchat/open/eligibility"],"inner_page":{"url":"https://www.paypal.com/ae/cshelp/article/can-i-archive-delete-or-cancel-an-invoice-help312?locale.x=ar_AE","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.paypalobjects.com/digitalassets/c/website/marketing/global/sitemap/paypal-sitemap-ar-ae.xml"}},"performance":{"response_ms":210,"bytes":367358,"redirects":1,"cookies_set":3},"why_this_score":[{"label":"Fastly detected","points":2},{"label":"DataDome seen by the browser","points":20}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/paypal.com.json"},{"schema":"ratradar.site/v1","host":"stripe.com","name":"Stripe","category":"finance","country":"United States","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://stripe.com/sitemap/sitemap.xml","Read the JSON-LD on the page (WebSite, Organization). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example stripe.com/cookie-settings/enforcement-mode. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":8,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://stripe.com/sitemap/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["WebSite","Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["stripe.com/cookie-settings/enforcement-mode","stripe.com/notifications","ext.stripe.com/universal-chat/agent-availability"],"inner_page":{"url":"https://stripe.com/newsroom/news/state-of-checkouts-2022","status":200,"json_ld_types":["Article","NewsArticle"],"skipped":null,"sitemap":"https://stripe.com/sitemap/partition-2.xml"}},"performance":{"response_ms":291,"bytes":400000,"redirects":0,"cookies_set":1},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/stripe.com.json"},{"schema":"ratradar.site/v1","host":"wise.com","name":"Wise","category":"finance","country":"United Kingdom","verdict":"open","hostility_score":8,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://wise.com/sitemap","Read the JSON-LD on the page (Organization). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example wise.com/bff/account-details/wise-qr/v1. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare","CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":12,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://wise.com/sitemap"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["wise.com/bff/account-details/wise-qr/v1","wise.com/gateway/v4/comparisons"],"inner_page":{"url":"https://wise.com/br/business-tools/cnae/4120400","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://wise.com/landing-pages/sitemap.xml"}},"performance":{"response_ms":917,"bytes":400000,"redirects":0,"cookies_set":3},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/wise.com.json"},{"schema":"ratradar.site/v1","host":"www.crunchbase.com","name":"Crunchbase","category":"finance","country":"United States","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt."],"risks":["robots.txt forbids 10 of 14 known AI crawlers (GPTBot, ClaudeBot, Claude-Web, anthropic-ai, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":["ratelimit-limit","ratelimit-policy","ratelimit-remaining","ratelimit-reset"],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":17,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ClaudeBot","Claude-Web","anthropic-ai","CCBot","Google-Extended","Bytespider","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":10,"sitemap_urls":["https://www.crunchbase.com/www-sitemaps/sitemap-index.xml"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":78,"bytes":129686,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 10 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/crunchbase.com.json"},{"schema":"ratradar.site/v1","host":"pitchbook.com","name":"PitchBook","category":"finance","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":7,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://pitchbook.com/sitemap.xml"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":204,"bytes":5532,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/pitchbook.com.json"},{"schema":"ratradar.site/v1","host":"opencorporates.com","name":"OpenCorporates","category":"finance","country":"United Kingdom","verdict":"open","hostility_score":8,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","The site publishes 2 RSS or Atom feeds. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example opencorporates.com/users/who-am-i. Untested, check the terms before using them."],"risks":["robots.txt forbids some AI crawlers: GPTBot."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront","Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"AmazonS3"},"robots":{"found":true,"disallow_all":false,"disallow_rules":13,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot"],"ai_crawlers_blocked_count":1,"sitemap_urls":["https://opencorporates.com/sitemap.xml.gz"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":2,"feed_urls":["https://opencorporates.com/feed/","https://opencorporates.com/comments/feed/"],"api_endpoints_seen":["opencorporates.com/users/who-am-i","js.zi-scripts.com/unified/v1/master/getSubscriptions"],"inner_page":null},"performance":{"response_ms":1110,"bytes":195196,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"CloudFront detected","points":2},{"label":"Cloudflare seen by the browser","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 1 AI crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/opencorporates.com.json"},{"schema":"ratradar.site/v1","host":"find-and-update.company-information.service.gov.uk","name":"Companies House","category":"gov","country":"United Kingdom","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"CompaniesHouse"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":765,"bytes":32688,"redirects":0,"cookies_set":2},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/find-and-update.company-information.service.gov.uk.json"},{"schema":"ratradar.site/v1","host":"www.dnb.com","name":"Dun & Bradstreet","category":"finance","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Enumerate pages from the sitemap: https://www.dnb.com/sitemapindex.xml","Read the JSON-LD on the page (Organization, WebPage, WebSite). It is cleaner than the markup."],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare","Akamai","Fastly","CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":48,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.dnb.com/sitemapindex.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["Organization","WebPage","WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.dnb.com/en-us/products/dnb-connect/dnb-connect-discover.html","status":200,"json_ld_types":["WebPage","WebSite","Organization"],"skipped":null,"sitemap":"https://www.dnb.com/en-us/sitemap.xml"}},"performance":{"response_ms":180,"bytes":116455,"redirects":1,"cookies_set":2},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"Akamai detected","points":10},{"label":"Fastly detected","points":2},{"label":"CloudFront detected","points":2},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/dnb.com.json"},{"schema":"ratradar.site/v1","host":"www.zoominfo.com","name":"ZoomInfo","category":"finance","country":"United States","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.zoominfo.com/cws/general-sitemap.xml","Read the JSON-LD on the page (WebSite, Organization, SoftwareApplication, FAQPage). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.zoominfo.com/api/contentful. Untested, check the terms before using them."],"risks":["robots.txt forbids 4 of 14 known AI crawlers (CCBot, Bytespider, Amazonbot, Meta-ExternalAgent). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":55,"crawl_delay_seconds":null,"ai_crawlers_blocked":["CCBot","Bytespider","Amazonbot","Meta-ExternalAgent"],"ai_crawlers_blocked_count":4,"sitemap_urls":["https://www.zoominfo.com/cws/general-sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["WebSite","Organization","SoftwareApplication","FAQPage"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.zoominfo.com/api/contentful","experience.ninetailed.co/v2/organizations/c06b94ae-235a-4601-a087-3e1c4d02502e/environments/main/profiles","cdn.dreamdata.cloud/api/v1/p","events.navattic.com/api/visitors/config","play.vidyard.com/player/ErGrtZFzUu5hSc49CXScv4.json"],"inner_page":null},"performance":{"response_ms":528,"bytes":339504,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 4 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/zoominfo.com.json"},{"schema":"ratradar.site/v1","host":"www.capterra.com","name":"Capterra","category":"reviews","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies.","Enumerate pages from the sitemap: https://www.capterra.com/sitemaps/sitemap.xml"],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans.","robots.txt forbids some AI crawlers: Bytespider."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":19,"crawl_delay_seconds":null,"ai_crawlers_blocked":["Bytespider"],"ai_crawlers_blocked_count":1,"sitemap_urls":["https://www.capterra.com/sitemaps/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":46,"bytes":5535,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"strong","signals":[{"kind":"robots_txt","label":"robots.txt blocks 1 AI crawler"},{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/capterra.com.json"},{"schema":"ratradar.site/v1","host":"www.g2.com","name":"G2","category":"reviews","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans.","robots.txt forbids some AI crawlers: Bytespider."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare","DataDome"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":88,"crawl_delay_seconds":null,"ai_crawlers_blocked":["Bytespider"],"ai_crawlers_blocked_count":1,"sitemap_urls":["https://www.g2.com/sitemaps/sitemap_index.xml.gz","https://www.g2.com/sitemaps/sitemap_index_compare.xml.gz","https://www.g2.com/llms.txt"]},"machine_readable":{"sitemap":false,"llms_txt":true,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":64,"bytes":1707,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6},{"label":"DataDome detected","points":20}],"eu_ai_training":{"level":"strong","signals":[{"kind":"robots_txt","label":"robots.txt blocks 1 AI crawler"},{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/g2.com.json"},{"schema":"ratradar.site/v1","host":"www.statista.com","name":"Statista","category":"finance","country":"Germany","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.statista.com/sitemap/","A browser load called JSON endpoints that look like data, for example cdn.statcdn.com/static/js/userlikewidget_us.json. Untested, check the terms before using them."],"risks":["robots.txt forbids 12 of 14 known AI crawlers (GPTBot, ClaudeBot, Claude-Web, anthropic-ai, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":68,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ClaudeBot","Claude-Web","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Bytespider","Amazonbot","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":12,"sitemap_urls":["https://www.statista.com/sitemap/","https://de.statista.com/sitemap/"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["cdn.statcdn.com/static/js/userlikewidget_us.json","www.statista.com/news-feed-slider"],"inner_page":{"url":"https://www.statista.com/chart/19955/household-debt-balance-in-the-united-states/","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.statista.com/sitemap/index/google-news/"}},"performance":{"response_ms":592,"bytes":366422,"redirects":0,"cookies_set":3},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 12 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/statista.com.json"},{"schema":"ratradar.site/v1","host":"www.usa.gov","name":"USA.gov","category":"gov","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","robots.txt asks for a crawl delay of 10 seconds.","Enumerate pages from the sitemap: https://www.usa.gov/sitemap.xml","Read the JSON-LD on the page (WebSite, Organization). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example s3-us-gov-west-1.amazonaws.com/cg-4d6fb302-315f-403e-a96e-a7563ccddd3d/1.0/waittime.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":10,"crawl_delay_seconds":10,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.usa.gov/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebSite","Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["s3-us-gov-west-1.amazonaws.com/cg-4d6fb302-315f-403e-a96e-a7563ccddd3d/1.0/waittime.json"],"inner_page":{"url":"https://www.usa.gov/es/articulos/beneficios-del-gobierno-para-usted-y-su-familia","status":200,"json_ld_types":["Organization"],"skipped":null,"sitemap":"https://www.usa.gov/sitemap.xml"}},"performance":{"response_ms":41,"bytes":48647,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/usa.gov.json"},{"schema":"ratradar.site/v1","host":"data.gov","name":"Data.gov","category":"gov","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://data.gov/sitemap.xml"],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://data.gov/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://data.gov/maritime/p24529331/","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://data.gov/sitemap.xml"}},"performance":{"response_ms":59,"bytes":121847,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/data.gov.json"},{"schema":"ratradar.site/v1","host":"www.census.gov","name":"US Census","category":"gov","country":"United States","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.census.gov/sitemapindex/sitemap.xml","A browser load called JSON endpoints that look like data, for example www.census.gov/libs/granite/csrf/token.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":1,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.census.gov/sitemapindex/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.census.gov/libs/granite/csrf/token.json","www.census.gov/content/census/en/jcr:content/root/responsivegrid/calendarview_copy_co.event.json","data.census.gov/api/notifications","www.census.gov/main/resources/js/global-menu.json","www.census.gov/populationwidget/cron/data/pop.json"],"inner_page":{"url":"https://www.census.gov/data/datasets/time-series/demo.html","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.census.gov/sitemapindex/datasets.xml"}},"performance":{"response_ms":484,"bytes":386889,"redirects":0,"cookies_set":4},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/census.gov.json"},{"schema":"ratradar.site/v1","host":"www.bls.gov","name":"BLS","category":"gov","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Enumerate pages from the sitemap: https://www.bls.gov/sitemap.xml","The site publishes 2 RSS or Atom feeds. Prefer them for new items."],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":8,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.bls.gov/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":2,"feed_urls":["https://www.bls.gov/feed/bls_latest.rss","https://www.bls.gov/feed/bls_latest.rss"],"api_endpoints_seen":[],"inner_page":{"url":"https://www.bls.gov/productivity/videos/home.htm","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.bls.gov/productivity/sitemap-productivity.xml"}},"performance":{"response_ms":53,"bytes":78785,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/bls.gov.json"},{"schema":"ratradar.site/v1","host":"www.cdc.gov","name":"CDC","category":"gov","country":"United States","verdict":"open","hostility_score":10,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.cdc.gov/wcms-auto-sitemap-index.xml","A browser load called JSON endpoints that look like data, for example www.cdc.gov/JScript/surveys/redcap-survey-configs.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":39,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.cdc.gov/wcms-auto-sitemap-index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.cdc.gov/JScript/surveys/redcap-survey-configs.json"],"inner_page":{"url":"https://www.cdc.gov/pinkbook/hcp/table-of-contents/index.html","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.cdc.gov/pinkbook/wcms-auto-sitemap.xml"}},"performance":{"response_ms":69,"bytes":60803,"redirects":1,"cookies_set":2},"why_this_score":[{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/cdc.gov.json"},{"schema":"ratradar.site/v1","host":"www.nih.gov","name":"NIH","category":"gov","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Enumerate pages from the sitemap: https://www.nih.gov/sitemap.xml","The site publishes 1 RSS or Atom feed. Prefer them for new items."],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":34,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":1,"feed_urls":["https://www.nih.gov/rss.xml"],"api_endpoints_seen":[],"inner_page":{"url":"https://www.nih.gov/research-training/medical-research-initiatives/include-project/events/include-project-clinical-trials-seminar-series-value-diverse-perspectives-down-syndrome-research","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.nih.gov/sitemap.xml"}},"performance":{"response_ms":336,"bytes":63114,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/nih.gov.json"},{"schema":"ratradar.site/v1","host":"www.who.int","name":"WHO","category":"gov","country":"Switzerland","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: http://www.who.int/sitemaps/sitemapindex.xml","Read the JSON-LD on the page (WebSite, Organization, WebPage). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.who.int/api/news/newsitems. Untested, check the terms before using them."],"risks":["robots.txt forbids some AI crawlers: CCBot."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":["CCBot"],"ai_crawlers_blocked_count":1,"sitemap_urls":["http://www.who.int/sitemaps/sitemapindex.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebSite","Organization","WebPage"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.who.int/api/news/newsitems","heatmaps.monsido.com/v1/settings/KXrCxamCowqmhCDxsvBllw.json"],"inner_page":{"url":"https://www.who.int/news-room/events/international-food-safety-conference/logistics-note","status":200,"json_ld_types":["BreadcrumbList","WebPage"],"skipped":null,"sitemap":"http://www.who.int/sitemap/sitemap.gz"}},"performance":{"response_ms":55,"bytes":138642,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 1 AI crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/who.int.json"},{"schema":"ratradar.site/v1","host":"www.nasa.gov","name":"NASA","category":"gov","country":"United States","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.nasa.gov/sitemap.xml","Read the JSON-LD on the page (Organization, WebSite, ImageObject, WebPage). It is cleaner than the markup.","The site publishes 1 RSS or Atom feed. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example plus.nasa.gov/wp-json/nasaplus/v1/live. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["Organization","WebSite","ImageObject","WebPage","Person","Article","VideoObject"],"feeds":1,"feed_urls":["https://www.nasa.gov/feed/"],"api_endpoints_seen":["plus.nasa.gov/wp-json/nasaplus/v1/live","api.parsely.com/v2/profile"],"inner_page":{"url":"https://science.nasa.gov/missions/hubble/hubble-achieves-milestone-100000th-exposure/","status":200,"json_ld_types":["Organization","WebSite","ImageObject","WebPage","Person","Article","NewsArticle"],"skipped":null,"sitemap":"https://www.nasa.gov/wp-sitemap-news-1.xml"}},"performance":{"response_ms":66,"bytes":356383,"redirects":0,"cookies_set":0},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/nasa.gov.json"},{"schema":"ratradar.site/v1","host":"www.noaa.gov","name":"NOAA","category":"gov","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.noaa.gov/sitemap.xml","The site publishes 1 RSS or Atom feed. Prefer them for new items."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"Apache"},"robots":{"found":true,"disallow_all":false,"disallow_rules":45,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":1,"feed_urls":["https://www.noaa.gov/rss.xml"],"api_endpoints_seen":[],"inner_page":{"url":"https://www.noaa.gov/regional-collaboration-network/regions-central/news/dummy-story-3","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.noaa.gov/sitemap.xml?page=1"}},"performance":{"response_ms":138,"bytes":50914,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/noaa.gov.json"},{"schema":"ratradar.site/v1","host":"www.usgs.gov","name":"USGS","category":"gov","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Enumerate pages from the sitemap: https://www.usgs.gov/sitemap.xml"],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare","CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":22,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.usgs.gov/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.usgs.gov/products/software/software-management","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.usgs.gov/sitemap.xml?page=1"}},"performance":{"response_ms":4783,"bytes":128217,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"CloudFront detected","points":2},{"label":"Slow response (4783 ms)","points":5},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/usgs.gov.json"},{"schema":"ratradar.site/v1","host":"sam.gov","name":"SAM.gov","category":"gov","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","A browser load called JSON endpoints that look like data, for example sam.gov/api/prod/alerts. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"istio-envoy"},"robots":{"found":true,"disallow_all":false,"disallow_rules":34,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["sam.gov/api/prod/alerts","sam.gov/toggles","jnn-pa.googleapis.com/$rpc/google.internal.waa.v1.Waa/GenerateIT"],"inner_page":null},"performance":{"response_ms":38,"bytes":55471,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/sam.gov.json"},{"schema":"ratradar.site/v1","host":"www.uspto.gov","name":"USPTO","category":"gov","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.uspto.gov/sitemap.xml","Read the JSON-LD on the page (GovernmentOrganization, WebSite). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.uspto.gov/alerts.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"Apache"},"robots":{"found":true,"disallow_all":false,"disallow_rules":5,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.uspto.gov/sitemap.xml","https://foiadocuments.uspto.gov/foia_sitemap.xml","https://ipidentifier.uspto.gov/ipidentifier_sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["GovernmentOrganization","WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.uspto.gov/alerts.json"],"inner_page":{"url":"https://www.uspto.gov/learning-and-resources/ip-policy/economic-research/events/2016-patentsview-workshop-engaging-user","status":404,"json_ld_types":[],"skipped":null,"sitemap":"https://www.uspto.gov/sitemap-1.xml"}},"performance":{"response_ms":328,"bytes":197702,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/uspto.gov.json"},{"schema":"ratradar.site/v1","host":"www.federalregister.gov","name":"Federal Register","category":"gov","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","A browser load called JSON endpoints that look like data, for example unblock.federalregister.gov/api/ip-management/v1/client-ip. Untested, check the terms before using them."],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":8,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.federalregister.gov/sitemaps/sitemap.xml.gz"]},"machine_readable":{"sitemap":false,"llms_txt":true,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["unblock.federalregister.gov/api/ip-management/v1/client-ip"],"inner_page":null},"performance":{"response_ms":247,"bytes":51676,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/federalregister.gov.json"},{"schema":"ratradar.site/v1","host":"www.congress.gov","name":"Congress.gov","category":"gov","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies.","robots.txt asks for a crawl delay of 2 seconds."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans.","robots.txt forbids 13 of 14 known AI crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":11,"crawl_delay_seconds":2,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","Claude-Web","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Bytespider","Amazonbot","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":13,"sitemap_urls":["https://www.congress.gov/sitemap/index.xml"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":26,"bytes":5556,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"strong","signals":[{"kind":"robots_txt","label":"robots.txt blocks 13 AI crawlers"},{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/congress.gov.json"},{"schema":"ratradar.site/v1","host":"www.gov.uk","name":"GOV.UK","category":"gov","country":"United Kingdom","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.gov.uk/sitemap.xml"],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":2,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.gov.uk/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.gov.uk/government/news/safety-bulletin-42024-published","status":200,"json_ld_types":["NewsArticle","BreadcrumbList"],"skipped":null,"sitemap":"https://www.gov.uk/sitemaps/sitemap_1.xml"}},"performance":{"response_ms":60,"bytes":85520,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/gov.uk.json"},{"schema":"ratradar.site/v1","host":"data.europa.eu","name":"EU Open Data","category":"gov","country":"Belgium","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://data.europa.eu/sitemap.xml","Read the JSON-LD on the page (GovernmentOrganization, WebSite). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example data.europa.eu/en/api/gettip. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":30,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://data.europa.eu/sitemap.xml","https://data.europa.eu/en/sitemap.xml","https://data.europa.eu/dashboard/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["GovernmentOrganization","WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["data.europa.eu/en/api/gettip"],"inner_page":{"url":"https://data.europa.eu/data/datasets/https-dane-gov-pl-pl-dataset-3989-db3c7fde3cbdf0b255a5062aee1cff6b8364811bbe8c94422453f27174ad029e?locale=en","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://data.europa.eu/data/sitemap_1.xml"}},"performance":{"response_ms":263,"bytes":112435,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/data.europa.eu.json"},{"schema":"ratradar.site/v1","host":"ec.europa.eu","name":"Eurostat","category":"gov","country":"Belgium","verdict":"open","hostility_score":17,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls.","Enumerate pages from the sitemap: https://ec.europa.eu/sitemap.xml","A browser load called JSON endpoints that look like data, for example eams.fpfis.tech.ec.europa.eu/config. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":199,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["eams.fpfis.tech.ec.europa.eu/config"],"inner_page":{"url":"https://ec.europa.eu/trade/trade-policy-and-you/events/","status":200,"json_ld_types":["BreadcrumbList","CreativeWork"],"skipped":null,"sitemap":"https://ec.europa.eu/trade/sitemap.xml"}},"performance":{"response_ms":1646,"bytes":16820,"redirects":1,"cookies_set":0},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/ec.europa.eu.json"},{"schema":"ratradar.site/v1","host":"www.worldbank.org","name":"World Bank","category":"gov","country":"United States","verdict":"open","hostility_score":8,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.worldbank.org/sitemap.xml","Read the JSON-LD on the page (Organization, WebSite, WebPage, BreadcrumbList). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.worldbank.org/ext/en/placeholders.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare","Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.worldbank.org/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["Organization","WebSite","WebPage","BreadcrumbList"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.worldbank.org/ext/en/placeholders.json","www.worldbank.org/ext/en/siteconfig/i18n.json","www.worldbank.org/ext/placeholders.json","worldbank.tt.omtrdc.net/m2/worldbank/mbox/json","webapi.worldbank.org/aemsite/wblive/global/search"],"inner_page":{"url":"https://www.worldbank.org/pt/news/press-release/2026/07/brazil-world-bank-group-backs-fiscal-reform-sustainable-urban-growth-fortaleza","status":200,"json_ld_types":["WebPage","NewsArticle","BreadcrumbList"],"skipped":null,"sitemap":"https://www.worldbank.org/content/vdam/sitemap/www-worldbank-org/en-news-sitemap-1.xml"}},"performance":{"response_ms":147,"bytes":61421,"redirects":1,"cookies_set":2},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/worldbank.org.json"},{"schema":"ratradar.site/v1","host":"www.imf.org","name":"IMF","category":"gov","country":"United States","verdict":"guarded","hostility_score":45,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":142,"bytes":48,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"HTTP 404 on a plain request","points":35},{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":404,"headless_browser_status":404,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/imf.org.json"},{"schema":"ratradar.site/v1","host":"www.oecd.org","name":"OECD","category":"gov","country":"France","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":2,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.oecd.org/sitemap.xml"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":35,"bytes":5552,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/oecd.org.json"},{"schema":"ratradar.site/v1","host":"www.un.org","name":"UN","category":"gov","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","robots.txt asks for a crawl delay of 10 seconds."],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"Apache"},"robots":{"found":true,"disallow_all":false,"disallow_rules":37,"crawl_delay_seconds":10,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":209,"bytes":9859,"redirects":0,"cookies_set":4},"why_this_score":[{"label":"CloudFront detected","points":2},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/un.org.json"},{"schema":"ratradar.site/v1","host":"www.ecb.europa.eu","name":"ECB","category":"gov","country":"Germany","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","robots.txt asks for a crawl delay of 5 seconds.","Enumerate pages from the sitemap: https://www.ecb.europa.eu/sitemap.xml","A browser load called JSON endpoints that look like data, for example www.ecb.europa.eu/shared/nav/navigation.min.en.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"myracloud"},"robots":{"found":true,"disallow_all":false,"disallow_rules":32,"crawl_delay_seconds":5,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.ecb.europa.eu/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.ecb.europa.eu/shared/nav/navigation.min.en.json"],"inner_page":{"url":"https://www.ecb.europa.eu/mopo/implement/app/html/index.en.html","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.ecb.europa.eu/sitemap.xml"}},"performance":{"response_ms":1853,"bytes":195403,"redirects":1,"cookies_set":0},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/ecb.europa.eu.json"},{"schema":"ratradar.site/v1","host":"www.data.gov.uk","name":"Data.gov.uk","category":"gov","country":"United Kingdom","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":"gunicorn"},"robots":{"found":true,"disallow_all":false,"disallow_rules":1,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":583,"bytes":21603,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/data.gov.uk.json"},{"schema":"ratradar.site/v1","host":"www.ons.gov.uk","name":"ONS","category":"gov","country":"United Kingdom","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Read the JSON-LD on the page (GovernmentOrganization). It is cleaner than the markup."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":["GovernmentOrganization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":256,"bytes":80858,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/ons.gov.uk.json"},{"schema":"ratradar.site/v1","host":"www.statcan.gc.ca","name":"Statistics Canada","category":"gov","country":"Canada","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":34,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":804,"bytes":389006,"redirects":0,"cookies_set":0},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/statcan.gc.ca.json"},{"schema":"ratradar.site/v1","host":"www.abs.gov.au","name":"ABS Australia","category":"gov","country":"Australia","verdict":"open","hostility_score":8,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.abs.gov.au/sitemap.xml"],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare","CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":5,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.abs.gov.au/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.abs.gov.au/census/census-media-hub/news/speeches","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.abs.gov.au/sitemap.xml?page=1"}},"performance":{"response_ms":202,"bytes":103906,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/abs.gov.au.json"},{"schema":"ratradar.site/v1","host":"www.clinicaltrialsregister.eu","name":"ClinicalTrials EU","category":"gov","country":"Belgium","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":540,"bytes":10293,"redirects":0,"cookies_set":3},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/clinicaltrialsregister.eu.json"},{"schema":"ratradar.site/v1","host":"www.ema.europa.eu","name":"EMA","category":"gov","country":"Netherlands","verdict":"open","hostility_score":8,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.ema.europa.eu/sitemap.xml","Read the JSON-LD on the page (Organization, WebSite). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example eams.fpfis.tech.ec.europa.eu/config. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare","CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":35,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.ema.europa.eu/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["Organization","WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["eams.fpfis.tech.ec.europa.eu/config"],"inner_page":null},"performance":{"response_ms":708,"bytes":94180,"redirects":1,"cookies_set":0},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/ema.europa.eu.json"},{"schema":"ratradar.site/v1","host":"go.drugbank.com","name":"DrugBank","category":"gov","country":"Canada","verdict":"blocked","hostility_score":76,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies.","Enumerate pages from the sitemap: https://go.drugbank.com/sitemap.xml"],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":3,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://go.drugbank.com/sitemap.xml","https://go.drugbank.com/sitemap-marketing.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":72,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/go.drugbank.com.json"},{"schema":"ratradar.site/v1","host":"www.webmd.com","name":"WebMD","category":"gov","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","A browser load called JSON endpoints that look like data, for example www.webmd.com/kapi/trends. Untested, check the terms before using them."],"risks":["Hostile to automation. Keep volumes low and cache.","robots.txt forbids 3 of 14 known AI crawlers (GPTBot, ClaudeBot, CCBot). An agent that identifies as an AI crawler should not fetch this site.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare","Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":22,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ClaudeBot","CCBot"],"ai_crawlers_blocked_count":3,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.webmd.com/kapi/trends","ssl.o.webmd.com/ee/v1/identity/acquire","www.webmd.com/kapi/secure/phydir/carefinder","ssl.o.webmd.com/ee/or2/v1/interact"],"inner_page":null},"performance":{"response_ms":234,"bytes":400000,"redirects":0,"cookies_set":5},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"Akamai detected","points":10},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"strong","signals":[{"kind":"robots_txt","label":"robots.txt blocks 3 AI crawlers"},{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/webmd.com.json"},{"schema":"ratradar.site/v1","host":"www.mayoclinic.org","name":"Mayo Clinic","category":"gov","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Akamai"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"AkamaiGHost"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":46,"bytes":370,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/mayoclinic.org.json"},{"schema":"ratradar.site/v1","host":"www.healthgrades.com","name":"Healthgrades","category":"reviews","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.healthgrades.com/sitemapindex.xml","A browser load called JSON endpoints that look like data, for example www.healthgrades.com/healthguides/ucms-api/hg-ucms-header. Untested, check the terms before using them."],"risks":["robots.txt forbids 9 of 14 known AI crawlers (GPTBot, ClaudeBot, anthropic-ai, CCBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":56,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ClaudeBot","anthropic-ai","CCBot","Google-Extended","Bytespider","Amazonbot","Applebot-Extended","Meta-ExternalAgent"],"ai_crawlers_blocked_count":9,"sitemap_urls":["https://www.healthgrades.com/sitemapindex.xml","https://www.healthgrades.com/sitemapindex-rlp.xml","https://www.healthgrades.com/sitemapindex-psr.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.healthgrades.com/healthguides/ucms-api/hg-ucms-header","www.healthgrades.com/api/autosuggest","www.healthgrades.com/api/mario/driver-admin/v4/drivers","taggy.rvohealth.com/implementations/public"],"inner_page":{"url":"https://www.healthgrades.com/providers/jami-hain-y56747z","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.healthgrades.com/fertility-reproductive-endocrinology-provider-profile-1.xml.gz"}},"performance":{"response_ms":1778,"bytes":400000,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 9 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/healthgrades.com.json"},{"schema":"ratradar.site/v1","host":"www.semanticscholar.org","name":"Semantic Scholar","category":"edu","country":"United States","verdict":"open","hostility_score":20,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.semanticscholar.org/sitemap_author_index.xml"],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["AWS WAF","CloudFront","Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":8,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.semanticscholar.org/sitemap_author_index.xml","https://www.semanticscholar.org/sitemap_paper_index.xml","https://www.semanticscholar.org/sitemap_topic_index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.semanticscholar.org/author/S.-Saalfeld/1889009","status":202,"json_ld_types":[],"skipped":null,"sitemap":"https://www.semanticscholar.org/sitemap-author-0000000.xml"}},"performance":{"response_ms":88,"bytes":188050,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"AWS WAF detected","points":12},{"label":"CloudFront detected","points":2},{"label":"Cloudflare seen by the browser","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/semanticscholar.org.json"},{"schema":"ratradar.site/v1","host":"www.researchgate.net","name":"ResearchGate","category":"edu","country":"Germany","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":7,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":38,"bytes":23066,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/researchgate.net.json"},{"schema":"ratradar.site/v1","host":"www.jstor.org","name":"JSTOR","category":"edu","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here."],"risks":["Hostile to automation. Keep volumes low and cache.","robots.txt forbids 4 of 14 known AI crawlers (GPTBot, ClaudeBot, CCBot, Google-Extended). An agent that identifies as an AI crawler should not fetch this site.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":19,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ClaudeBot","CCBot","Google-Extended"],"ai_crawlers_blocked_count":4,"sitemap_urls":["https://www.jstor.org/sitemap.xml"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":47,"bytes":3038,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"Fastly detected","points":2},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"strong","signals":[{"kind":"robots_txt","label":"robots.txt blocks 4 AI crawlers"},{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/jstor.org.json"},{"schema":"ratradar.site/v1","host":"www.coursera.org","name":"Coursera","category":"edu","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.coursera.org/sitemap.xml","A browser load called JSON endpoints that look like data, for example www.coursera.org/api/rest/v1/client-gateway/eval. Untested, check the terms before using them."],"risks":["robots.txt forbids some AI crawlers: Meta-ExternalAgent."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"envoy"},"robots":{"found":true,"disallow_all":false,"disallow_rules":23,"crawl_delay_seconds":null,"ai_crawlers_blocked":["Meta-ExternalAgent"],"ai_crawlers_blocked_count":1,"sitemap_urls":["https://www.coursera.org/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.coursera.org/api/rest/v1/client-gateway/eval","www.coursera.org/graphql-gateway","www.coursera.org/api/rest/v1/eventing/infobatch"],"inner_page":{"url":"https://www.coursera.org/learn/adobe-dreamweaver-cc-2017-lo092005","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.coursera.org/sitemap~www~courses.xml"}},"performance":{"response_ms":452,"bytes":400000,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 1 AI crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/coursera.org.json"},{"schema":"ratradar.site/v1","host":"www.khanacademy.org","name":"Khan Academy","category":"edu","country":"United States","verdict":"open","hostility_score":15,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls.","Enumerate pages from the sitemap: https://www.khanacademy.org/sitemap.xml","A browser load called JSON endpoints that look like data, for example www.khanacademy.org/_fs-ch-1T1wmsGaOgGaSxcX/fst-post-back. Untested, check the terms before using them."],"risks":["robots.txt forbids some AI crawlers: GPTBot."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":29,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot"],"ai_crawlers_blocked_count":1,"sitemap_urls":["https://www.khanacademy.org/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.khanacademy.org/_fs-ch-1T1wmsGaOgGaSxcX/fst-post-back","www.khanacademy.org/api/features/sdk-u16yaYPyqdZ0OHuZ","www.khanacademy.org/api/internal/graphql/localesForLangPickerQuery","www.khanacademy.org/api/internal/graphql/districtId","www.khanacademy.org/api/internal/graphql/learnMenuTopicsQuery","www.khanacademy.org/api/internal/graphql/footerDomainsQuery"],"inner_page":{"url":"https://www.khanacademy.org/math/mappers","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.khanacademy.org/sitemaps/math/mappers/sitemap.xml"}},"performance":{"response_ms":212,"bytes":238422,"redirects":0,"cookies_set":4},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 1 AI crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/khanacademy.org.json"},{"schema":"ratradar.site/v1","host":"www.udemy.com","name":"Udemy","category":"edu","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Read the JSON-LD on the page (WebSite). It is cleaner than the markup."],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":37,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.udemy.com/sitemap.xml"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":["WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":104,"bytes":303270,"redirects":0,"cookies_set":3},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/udemy.com.json"},{"schema":"ratradar.site/v1","host":"foursquare.com","name":"Foursquare","category":"reviews","country":"United States","verdict":"open","hostility_score":8,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://4sq-sitemap.s3.amazonaws.com/sitemap_index.xml","Read the JSON-LD on the page (WebPage, ImageObject, BreadcrumbList, WebSite). It is cleaner than the markup.","The site publishes 2 RSS or Atom feeds. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example api.parsely.com/v2/profile. Untested, check the terms before using them."],"risks":["robots.txt forbids some AI crawlers: GPTBot, ChatGPT-User."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly","Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":27,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User"],"ai_crawlers_blocked_count":2,"sitemap_urls":["https://4sq-sitemap.s3.amazonaws.com/sitemap_index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebPage","ImageObject","BreadcrumbList","WebSite","Organization"],"feeds":2,"feed_urls":["https://foursquare.com/feed/","https://foursquare.com/comments/feed/"],"api_endpoints_seen":["api.parsely.com/v2/profile","app.leandata.com/snippet-config","forms-eu1.hsforms.com/embed/v3/form/144648280/71cea14a-e9f8-4b6e-92c1-ecc26d2999af/json","forms-eu1.hsforms.com/embed/v3/form/144648280/66814d48-6cf0-48d8-b0a4-6941ec393eeb/json"],"inner_page":null},"performance":{"response_ms":47,"bytes":212627,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Fastly detected","points":2},{"label":"Cloudflare seen by the browser","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 2 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/foursquare.com.json"},{"schema":"ratradar.site/v1","host":"www.yellowpages.com","name":"Yellow Pages","category":"reviews","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":50,"bytes":5854,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/yellowpages.com.json"},{"schema":"ratradar.site/v1","host":"www.bbb.org","name":"BBB","category":"reviews","country":"United States","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","A browser load called JSON endpoints that look like data, for example www.bbb.org/api/suggest/location. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":20,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.bbb.org/sitemap-business-profiles-index.xml","https://www.bbb.org/sitemap-accredited-business-profiles-index.xml","https://www.bbb.org/sitemap-articles-index.xml"]},"machine_readable":{"sitemap":false,"llms_txt":true,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.bbb.org/api/suggest/location"],"inner_page":null},"performance":{"response_ms":336,"bytes":148794,"redirects":0,"cookies_set":14},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/bbb.org.json"},{"schema":"ratradar.site/v1","host":"www.angi.com","name":"Angi","category":"reviews","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies.","Enumerate pages from the sitemap: https://www.angi.com/sitemap/article-sitemap.xml"],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":13,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.angi.com/sitemap/article-sitemap.xml","https://www.angi.com/sitemap/geo-sitemap-index.xml","https://www.angi.com/sitemap/leaf-sitemap-index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":80,"bytes":5552,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/angi.com.json"},{"schema":"ratradar.site/v1","host":"www.thumbtack.com","name":"Thumbtack","category":"reviews","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Read the JSON-LD on the page (Website, Organization). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example app.thumbtack.com/graphql. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":32,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.thumbtack.com/sitemap_seo_showroom.xml","https://www.thumbtack.com/sitemap.xml"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":["Website","Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["app.thumbtack.com/graphql","cdn.thumbtackstatic.com/fe-assets-web/media/lottie/lottie_hp_btf_device_search.json","ct.pinterest.com/user/"],"inner_page":null},"performance":{"response_ms":450,"bytes":202127,"redirects":0,"cookies_set":5},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/thumbtack.com.json"},{"schema":"ratradar.site/v1","host":"www.opentable.com","name":"OpenTable","category":"food","country":"United States","verdict":"guarded","hostility_score":45,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":15064,"bytes":null,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"No answer to a plain request","points":40},{"label":"Slow response (15064 ms)","points":5}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":0,"headless_browser_status":0,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/opentable.com.json"},{"schema":"ratradar.site/v1","host":"www.doordash.com","name":"DoorDash","category":"food","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies.","Enumerate pages from the sitemap: https://www.doordash.com/sitemap-store-doordash-index.xml"],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":27,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.doordash.com/sitemap-store-doordash-index.xml","https://www.doordash.com/sitemap-business-doordash-index.xml","https://www.doordash.com/sitemap-business_menu-doordash-index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":55,"bytes":5599,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/doordash.com.json"},{"schema":"ratradar.site/v1","host":"www.grubhub.com","name":"Grubhub","category":"food","country":"United States","verdict":"guarded","hostility_score":39,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls.","A browser load called JSON endpoints that look like data, for example featureflags.grubhub.com/v1/sdkConfig. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly","CloudFront","PerimeterX"],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":"openresty"},"robots":{"found":true,"disallow_all":false,"disallow_rules":17,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["featureflags.grubhub.com/v1/sdkConfig"],"inner_page":null},"performance":{"response_ms":78,"bytes":14626,"redirects":0,"cookies_set":3},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15},{"label":"Fastly detected","points":2},{"label":"CloudFront detected","points":2},{"label":"PerimeterX seen by the browser","points":20}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/grubhub.com.json"},{"schema":"ratradar.site/v1","host":"www.ubereats.com","name":"Uber Eats","category":"food","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Read the JSON-LD on the page (WebSite). It is cleaner than the markup."],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":4,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.ubereats.com/sitemap-index-771af823.xml.gz","https://www.ubereats.com/sitemap-store-771af823-025.xml.gz","https://www.ubereats.com/sitemap-store-771af823-024.xml.gz"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":["WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":1384,"bytes":400000,"redirects":0,"cookies_set":10},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/ubereats.com.json"},{"schema":"ratradar.site/v1","host":"deliveroo.co.uk","name":"Deliveroo","category":"food","country":"United Kingdom","verdict":"guarded","hostility_score":26,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429.","Enumerate pages from the sitemap: https://deliveroo.co.uk/en/sitemap-index.xml","Read the JSON-LD on the page (Website). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example api.uk.deliveroo.com/consumer/basket/graphql. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare","PerimeterX"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":25,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://deliveroo.co.uk/en/sitemap-index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["Website"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["api.uk.deliveroo.com/consumer/basket/graphql","deliveroo.co.uk/api/auth/props"],"inner_page":{"url":"https://deliveroo.co.uk/more/news-articles/archive/2018","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://deliveroo.co.uk/en/sitemap-others-landing-pages-1.xml"}},"performance":{"response_ms":1076,"bytes":230830,"redirects":0,"cookies_set":6},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"PerimeterX seen by the browser","points":20}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/deliveroo.co.uk.json"},{"schema":"ratradar.site/v1","host":"www.just-eat.co.uk","name":"Just Eat","category":"food","country":"United Kingdom","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Read the JSON-LD on the page (WebSite, Organization). It is cleaner than the markup."],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare","Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":["ratelimit-limit","ratelimit-remaining","ratelimit-reset","x-ratelimit-limit-second","x-ratelimit-remaining-second"],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":27,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":["WebSite","Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":1722,"bytes":400000,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"Fastly detected","points":2},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/just-eat.co.uk.json"},{"schema":"ratradar.site/v1","host":"www.swiggy.com","name":"Swiggy","category":"food","country":"India","verdict":"hostile","hostility_score":59,"recommended_approach":"real_browser","steps":["Use a rendering browser, slow requests, and rotating proxies."],"risks":["Hostile to automation. Keep volumes low and cache."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":false},"protections":{"vendors":["AWS WAF","CloudFront"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"CloudFront"},"robots":{"found":true,"disallow_all":false,"disallow_rules":19,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.swiggy.com/sitemap.xml.gz","https://www.swiggy.com/scenes/sitemap/sitemap.xml.gz"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":27,"bytes":2423,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"A challenge page appeared","points":45},{"label":"AWS WAF detected","points":12},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":202,"headless_browser_status":0,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/swiggy.com.json"},{"schema":"ratradar.site/v1","host":"www.meituan.com","name":"Meituan","category":"food","country":"China","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","A browser load called JSON endpoints that look like data, for example catfront.dianping.com/api/pv. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"openresty"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["catfront.dianping.com/api/pv"],"inner_page":null},"performance":{"response_ms":638,"bytes":124475,"redirects":0,"cookies_set":0},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/meituan.com.json"},{"schema":"ratradar.site/v1","host":"www.allrecipes.com","name":"AllRecipes","category":"food","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Enumerate pages from the sitemap: https://www.allrecipes.com/sitemap.xml","Read the JSON-LD on the page (WebPage). It is cleaner than the markup.","The site publishes 1 RSS or Atom feed. Prefer them for new items."],"risks":["Hostile to automation. Keep volumes low and cache.","robots.txt forbids 11 of 14 known AI crawlers (ClaudeBot, Claude-Web, anthropic-ai, CCBot, ...). An agent that identifies as an AI crawler should not fetch this site.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":3,"crawl_delay_seconds":null,"ai_crawlers_blocked":["ClaudeBot","Claude-Web","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Bytespider","Amazonbot","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":11,"sitemap_urls":["https://www.allrecipes.com/sitemap.xml","https://www.allrecipes.com/google-news-sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebPage"],"feeds":1,"feed_urls":["https://feeds-api.dotdashmeredith.com/v1/rss/google/afd5e9ea-c220-419e-9135-d8457772e240"],"api_endpoints_seen":[],"inner_page":{"url":"https://www.allrecipes.com/prego-new-butternut-squash-alfredo-pasta-sauce-12163281","status":200,"json_ld_types":["NewsArticle"],"skipped":null,"sitemap":"https://www.allrecipes.com/google-news-sitemap.xml"}},"performance":{"response_ms":310,"bytes":400000,"redirects":0,"cookies_set":6},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 11 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/allrecipes.com.json"},{"schema":"ratradar.site/v1","host":"www.epicurious.com","name":"Epicurious","category":"food","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.epicurious.com/sitemap.xml","Read the JSON-LD on the page (Organization, WebSite, ItemList). It is cleaner than the markup.","The site publishes 1 RSS or Atom feed. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example www.epicurious.com/journey/compiler/latest.json. Untested, check the terms before using them."],"risks":["robots.txt forbids 9 of 14 known AI crawlers (ClaudeBot, CCBot, PerplexityBot, Google-Extended, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"CloudFront"},"robots":{"found":true,"disallow_all":false,"disallow_rules":16,"crawl_delay_seconds":null,"ai_crawlers_blocked":["ClaudeBot","CCBot","PerplexityBot","Google-Extended","Bytespider","Amazonbot","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":9,"sitemap_urls":["https://www.epicurious.com/sitemap.xml","https://www.epicurious.com/tagpages-sitemap.xml","https://www.epicurious.com/bundles-sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["Organization","WebSite","ItemList"],"feeds":1,"feed_urls":["https://www.epicurious.com/feed/rss"],"api_endpoints_seen":["www.epicurious.com/journey/compiler/latest.json","www.epicurious.com/auth/refresh","cdn.gladly.com/orgs/configs/chat/condenast.com-Epicurious.json"],"inner_page":null},"performance":{"response_ms":114,"bytes":400000,"redirects":0,"cookies_set":5},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 9 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/epicurious.com.json"},{"schema":"ratradar.site/v1","host":"www.youtube.com","name":"YouTube","category":"media","country":"United States","verdict":"open","hostility_score":20,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.youtube.com/sitemaps/sitemap.xml","A browser load called JSON endpoints that look like data, for example www.youtube.com/youtubei/v1/guide. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["PerimeterX"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Google","server":"ESF"},"robots":{"found":true,"disallow_all":false,"disallow_rules":21,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.youtube.com/sitemaps/sitemap.xml","https://www.youtube.com/product/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.youtube.com/youtubei/v1/guide","www.youtube.com/youtubei/v1/feedback","www.youtube.com/api/jnn/v1/GenerateIT"],"inner_page":{"url":"https://www.youtube.com/product/E_g_11h_b22912","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.youtube.com/product/sitemap-files/sitemap-001.xml.gz"}},"performance":{"response_ms":256,"bytes":400000,"redirects":0,"cookies_set":8},"why_this_score":[{"label":"PerimeterX detected","points":20}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/youtube.com.json"},{"schema":"ratradar.site/v1","host":"www.netflix.com","name":"Netflix","category":"media","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"respect_robots_skip","steps":["robots.txt disallows every crawler. Do not scrape it. Look for an official API, a data license, or a partner feed.","Enumerate pages from the sitemap: https://www.netflix.com/sitemap/index","A browser load called JSON endpoints that look like data, for example www.netflix.com/graphql. Untested, check the terms before using them."],"risks":["Scraping against an explicit robots.txt ban is the quickest way to a legal or blocking problem.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":false,"official_api_advised":true},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"envoy"},"robots":{"found":true,"disallow_all":true,"disallow_rules":1,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.netflix.com/sitemap/index"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.netflix.com/graphql"],"inner_page":null},"performance":{"response_ms":707,"bytes":400000,"redirects":0,"cookies_set":5},"why_this_score":[{"label":"robots.txt disallows everything","points":10},{"label":"A headless browser was refused though a plain request worked","points":0},{"label":"Raised to at least 50 because robots.txt disallows everything","points":0}],"eu_ai_training":{"level":"strong","signals":[{"kind":"robots_txt","label":"robots.txt bans every crawler"},{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/netflix.com.json"},{"schema":"ratradar.site/v1","host":"open.spotify.com","name":"Spotify","category":"media","country":"Sweden","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://open.spotify.com/sitemaps/sitemap-index.xml","A browser load called JSON endpoints that look like data, for example open.spotify.com/api/token. Untested, check the terms before using them."],"risks":["robots.txt forbids 10 of 14 known AI crawlers (GPTBot, ClaudeBot, Claude-Web, anthropic-ai, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":"envoy"},"robots":{"found":true,"disallow_all":false,"disallow_rules":3,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ClaudeBot","Claude-Web","anthropic-ai","CCBot","Google-Extended","Bytespider","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":10,"sitemap_urls":["https://open.spotify.com/sitemaps/sitemap-index.xml","https://open.spotify.com/sitemaps/sitemap-pages.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["open.spotify.com/api/token","apresolve.spotify.com/","clienttoken.spotify.com/v1/clienttoken","guc3-spclient.spotify.com/remote-config-resolver/v3/unauth/configuration","guc3-spclient.spotify.com/gabo-receiver-service/public/v3/events"],"inner_page":{"url":"https://open.spotify.com/intl-id/","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://open.spotify.com/sitemaps/sitemap-pages.xml"}},"performance":{"response_ms":329,"bytes":310304,"redirects":0,"cookies_set":5},"why_this_score":[{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 10 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/open.spotify.com.json"},{"schema":"ratradar.site/v1","host":"soundcloud.com","name":"SoundCloud","category":"media","country":"Germany","verdict":"guarded","hostility_score":37,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls.","Enumerate pages from the sitemap: https://soundcloud.com/sitemap.xml","A browser load called JSON endpoints that look like data, for example dwt.soundcloud.com/js/. Untested, check the terms before using them."],"risks":["robots.txt forbids 14 of 14 known AI crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["DataDome","CloudFront"],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"am/2"},"robots":{"found":true,"disallow_all":false,"disallow_rules":7,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","Claude-Web","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Bytespider","Amazonbot","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":14,"sitemap_urls":["https://soundcloud.com/sitemap.xml","https://soundcloud.com/sitemapIndex.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["dwt.soundcloud.com/js/","api-auth.soundcloud.com/oauth/session","soundcloud.com/versions.json","api-v2.soundcloud.com/announcements"],"inner_page":{"url":"https://soundcloud.com/pages/copyright/report","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://soundcloud.com/sitemap.xml"}},"performance":{"response_ms":280,"bytes":69062,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15},{"label":"DataDome detected","points":20},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 14 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/soundcloud.com.json"},{"schema":"ratradar.site/v1","host":"bandcamp.com","name":"Bandcamp","category":"media","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","A browser load called JSON endpoints that look like data, for example bandcamp.com/api/design_system/1/editorial_recommendations. Untested, check the terms before using them."],"risks":["robots.txt forbids 7 of 14 known AI crawlers (GPTBot, ClaudeBot, CCBot, Google-Extended, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":10,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ClaudeBot","CCBot","Google-Extended","Bytespider","Amazonbot","Meta-ExternalAgent"],"ai_crawlers_blocked_count":7,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["bandcamp.com/api/design_system/1/editorial_recommendations","bandcamp.com/api/radio_api/1/get_radio_shows","bandcamp.com/api/live_events_public_api/4/get_upcoming_live_events","bandcamp.com/api/salesfeed/1/get_initial","bandcamp.com/api/homepage_api/1/notable_tralbums_data","bandcamp.com/api/editorial/3/get_aotd"],"inner_page":null},"performance":{"response_ms":219,"bytes":96205,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 7 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/bandcamp.com.json"},{"schema":"ratradar.site/v1","host":"www.imdb.com","name":"IMDb","category":"media","country":"United States","verdict":"hostile","hostility_score":69,"recommended_approach":"respect_robots_skip","steps":["robots.txt disallows every crawler. Do not scrape it. Look for an official API, a data license, or a partner feed."],"risks":["Scraping against an explicit robots.txt ban is the quickest way to a legal or blocking problem."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":false,"official_api_advised":true},"protections":{"vendors":["AWS WAF","CloudFront"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"Server"},"robots":{"found":true,"disallow_all":true,"disallow_rules":1,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":135,"bytes":1997,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"A challenge page appeared","points":45},{"label":"AWS WAF detected","points":12},{"label":"CloudFront detected","points":2},{"label":"robots.txt disallows everything","points":10},{"label":"Raised to at least 50 because robots.txt disallows everything","points":0}],"eu_ai_training":{"level":"strong","signals":[{"kind":"robots_txt","label":"robots.txt bans every crawler"},{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":202,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/imdb.com.json"},{"schema":"ratradar.site/v1","host":"www.rottentomatoes.com","name":"Rotten Tomatoes","category":"reviews","country":"United States","verdict":"open","hostility_score":10,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.rottentomatoes.com/sitemaps/sitemap.xml","A browser load called JSON endpoints that look like data, for example app.mps.vsnt.net/request/page/json/params/. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":5,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.rottentomatoes.com/sitemaps/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["app.mps.vsnt.net/request/page/json/params/","www.rottentomatoes.com/napi/preferences/themes","www.rottentomatoes.com/napi/bridge/v1/wordpress/header-mobile"],"inner_page":{"url":"https://www.rottentomatoes.com/m/1221483-paa/pictures","status":null,"json_ld_types":[],"skipped":"robots.txt disallows this path","sitemap":"https://www.rottentomatoes.com/sitemaps/movie_0.xml"}},"performance":{"response_ms":69,"bytes":400000,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/rottentomatoes.com.json"},{"schema":"ratradar.site/v1","host":"letterboxd.com","name":"Letterboxd","category":"reviews","country":"New Zealand","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans.","robots.txt forbids 7 of 14 known AI crawlers (GPTBot, ClaudeBot, CCBot, Google-Extended, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":12,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ClaudeBot","CCBot","Google-Extended","Bytespider","Applebot-Extended","Meta-ExternalAgent"],"ai_crawlers_blocked_count":7,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":28,"bytes":5554,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"strong","signals":[{"kind":"robots_txt","label":"robots.txt blocks 7 AI crawlers"},{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/letterboxd.com.json"},{"schema":"ratradar.site/v1","host":"www.metacritic.com","name":"Metacritic","category":"reviews","country":"United States","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.metacritic.com/news/latest.xml","A browser load called JSON endpoints that look like data, for example services.fandom.com/geoip/location. Untested, check the terms before using them."],"risks":["robots.txt forbids 3 of 14 known AI crawlers (GPTBot, OAI-SearchBot, CCBot). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":7,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","OAI-SearchBot","CCBot"],"ai_crawlers_blocked_count":3,"sitemap_urls":["https://www.metacritic.com/news/latest.xml","https://www.metacritic.com/news/index.xml","https://www.metacritic.com/galleries/index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["services.fandom.com/geoip/location","services.fandom.com/galactus-user-profile/v2/profiles/me","www.metacritic.com/_i18n/c8b6fc72/en/messages.json","geo.privacymanager.io/","id5-sync.com/api/config/prebid","api.btmessage.com/websiteconfig"],"inner_page":{"url":"https://www.metacritic.com/news/2026-sundance-film-festival-recap-reviews-awards/","status":200,"json_ld_types":["NewsArticle"],"skipped":null,"sitemap":"https://www.metacritic.com/news/2026-01.xml"}},"performance":{"response_ms":80,"bytes":400000,"redirects":0,"cookies_set":5},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 3 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/metacritic.com.json"},{"schema":"ratradar.site/v1","host":"genius.com","name":"Genius","category":"media","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here."],"risks":["Hostile to automation. Keep volumes low and cache.","robots.txt forbids 10 of 14 known AI crawlers (GPTBot, ChatGPT-User, ClaudeBot, anthropic-ai, ...). An agent that identifies as an AI crawler should not fetch this site.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":18,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","ClaudeBot","anthropic-ai","CCBot","Google-Extended","Bytespider","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":10,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":74,"bytes":312572,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 10 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/genius.com.json"},{"schema":"ratradar.site/v1","host":"vimeo.com","name":"Vimeo","category":"media","country":"United States","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","robots.txt asks for a crawl delay of 2 seconds.","Enumerate pages from the sitemap: https://vimeo.com/marketing.xml.gz","A browser load called JSON endpoints that look like data, for example transcend-cdn.com/cm/31e97cbf-355c-4f0d-8bad-9cd945121508/translations/en.json. Untested, check the terms before using them."],"risks":["robots.txt forbids 7 of 14 known AI crawlers (GPTBot, ClaudeBot, Claude-Web, anthropic-ai, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":325,"crawl_delay_seconds":2,"ai_crawlers_blocked":["GPTBot","ClaudeBot","Claude-Web","anthropic-ai","CCBot","Google-Extended","Bytespider"],"ai_crawlers_blocked_count":7,"sitemap_urls":["https://vimeo.com/marketing.xml.gz","https://vimeo.com/sitemap/videos.xml.gz"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["transcend-cdn.com/cm/31e97cbf-355c-4f0d-8bad-9cd945121508/translations/en.json"],"inner_page":{"url":"https://vimeo.com/product/ad-free-video-hosting","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://vimeo.com/marketing/product.xml.gz"}},"performance":{"response_ms":60,"bytes":400000,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 7 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/vimeo.com.json"},{"schema":"ratradar.site/v1","host":"www.dailymotion.com","name":"Dailymotion","category":"media","country":"France","verdict":"open","hostility_score":17,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls.","Enumerate pages from the sitemap: https://www.dailymotion.com/map-videos-latest.xml"],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"DMS/2"},"robots":{"found":true,"disallow_all":false,"disallow_rules":107,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.dailymotion.com/map-videos-latest.xml","https://www.dailymotion.com/map-videos-default-strategic.xml","https://www.dailymotion.com/map-videos-default-partner.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.dailymotion.com/video/x8ctaop","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.dailymotion.com/map-videos-latest-000000.xml.gz"}},"performance":{"response_ms":350,"bytes":58120,"redirects":0,"cookies_set":4},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/dailymotion.com.json"},{"schema":"ratradar.site/v1","host":"www.bilibili.com","name":"Bilibili","category":"media","country":"China","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.bilibili.com/sitemap.xml","A browser load called JSON endpoints that look like data, for example api.bilibili.com/x/web-interface/nav. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":2,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["api.bilibili.com/x/web-interface/nav","i0.hdslb.com/bfs/banner/410a07d47d7f65fcc5429477c337ad1f892672f6.jpg@.avg_color","i0.hdslb.com/bfs/banner/53b2e7232ecbc86594e3eebc160692f914e43867.png@.avg_color","i0.hdslb.com/bfs/banner/8e3cbc49b8164df3d581087fad09b28a8f954415.jpg@.avg_color","i0.hdslb.com/bfs/banner/8b651214d64f6f7af99cf42d291146c60c82caff.jpg@.avg_color","i0.hdslb.com/bfs/banner/cffae6213d0d4edc228926f80988136e158226c2.jpg@.avg_color"],"inner_page":{"url":"http://www.bilibili.com/read/cv53387342/","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.bilibili.com/sitemap/read/detail.xml"}},"performance":{"response_ms":502,"bytes":169928,"redirects":0,"cookies_set":2},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/bilibili.com.json"},{"schema":"ratradar.site/v1","host":"www.nicovideo.jp","name":"Niconico","category":"media","country":"Japan","verdict":"open","hostility_score":8,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://nvfs.video.nicovideo.jp/google-video-sitemap/nicovideo_video_sitemap_index.xml","A browser load called JSON endpoints that look like data, for example wktk.nicovideo.jp/v2/wakutkool/frames.json. Untested, check the terms before using them."],"risks":["robots.txt forbids some AI crawlers: GPTBot."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront","Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"Apache"},"robots":{"found":true,"disallow_all":false,"disallow_rules":18,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot"],"ai_crawlers_blocked_count":1,"sitemap_urls":["https://nvfs.video.nicovideo.jp/google-video-sitemap/nicovideo_video_sitemap_index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["wktk.nicovideo.jp/v2/wakutkool/frames.json","common-header.nimg.jp/version/common_header_version.json","live2.nicovideo.jp/api/v2/programs/lv351485349/external.json","web-banner.ads.aps.amazon-adsystem.com/e/dtb/bid","secure-assets.rubiconproject.com/floors/1001-pbjs-floors.json"],"inner_page":{"url":"https://www.nicovideo.jp/watch/sm9","status":200,"json_ld_types":["VideoObject","WebSite","BreadcrumbList"],"skipped":null,"sitemap":"https://nvfs.video.nicovideo.jp/google-video-sitemap/nicovideo_video_sitemap.xml.0000000"}},"performance":{"response_ms":402,"bytes":149654,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"CloudFront detected","points":2},{"label":"Cloudflare seen by the browser","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 1 AI crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/nicovideo.jp.json"},{"schema":"ratradar.site/v1","host":"www.autotrader.com","name":"Autotrader","category":"auto","country":"United States","verdict":"guarded","hostility_score":25,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"AkamaiNetStorage"},"robots":{"found":true,"disallow_all":false,"disallow_rules":161,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":430,"bytes":3762,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15},{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/autotrader.com.json"},{"schema":"ratradar.site/v1","host":"www.cars.com","name":"Cars.com","category":"auto","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":28,"bytes":6022,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/cars.com.json"},{"schema":"ratradar.site/v1","host":"www.cargurus.com","name":"CarGurus","category":"auto","country":"United States","verdict":"guarded","hostility_score":28,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429.","Enumerate pages from the sitemap: https://www.cargurus.com/sitemap_index.xml","Read the JSON-LD on the page (Corporation, WebSite, WebPage). It is cleaner than the markup."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["DataDome","Fastly","Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":173,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.cargurus.com/sitemap_index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["Corporation","WebSite","WebPage"],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.cargurus.com/Cars/cl-Certified-2022-Chevrolet-Tahoe-c31981","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.cargurus.com/sitemap1.xml.gz"}},"performance":{"response_ms":469,"bytes":400000,"redirects":0,"cookies_set":4},"why_this_score":[{"label":"DataDome detected","points":20},{"label":"Fastly detected","points":2},{"label":"Cloudflare seen by the browser","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/cargurus.com.json"},{"schema":"ratradar.site/v1","host":"www.carvana.com","name":"Carvana","category":"auto","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Enumerate pages from the sitemap: https://www.carvana.com/sitemap.xml"],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare","PerimeterX"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":19,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.carvana.com/sitemap.xml","https://www.carvana.com/buy/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.carvana.com/cars/1990","status":403,"json_ld_types":[],"skipped":null,"sitemap":"https://www.carvana.com/sitemap_0.xml.gz"}},"performance":{"response_ms":291,"bytes":400000,"redirects":0,"cookies_set":16},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"PerimeterX detected","points":20},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/carvana.com.json"},{"schema":"ratradar.site/v1","host":"www.edmunds.com","name":"Edmunds","category":"auto","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies.","Enumerate pages from the sitemap: https://www.edmunds.com/sitemap.xml"],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans.","robots.txt forbids 4 of 14 known AI crawlers (GPTBot, ClaudeBot, Bytespider, Meta-ExternalAgent). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Akamai"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"AkamaiGHost"},"robots":{"found":true,"disallow_all":false,"disallow_rules":88,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ClaudeBot","Bytespider","Meta-ExternalAgent"],"ai_crawlers_blocked_count":4,"sitemap_urls":[]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":89,"bytes":16125,"redirects":0,"cookies_set":4},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"strong","signals":[{"kind":"robots_txt","label":"robots.txt blocks 4 AI crawlers"},{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/edmunds.com.json"},{"schema":"ratradar.site/v1","host":"www.kbb.com","name":"Kelley Blue Book","category":"auto","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Enumerate pages from the sitemap: https://www.kbb.com/sitemap-index.xml","Read the JSON-LD on the page (Organization). It is cleaner than the markup."],"risks":["Hostile to automation. Keep volumes low and cache.","robots.txt forbids some AI crawlers: Amazonbot.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":33,"crawl_delay_seconds":null,"ai_crawlers_blocked":["Amazonbot"],"ai_crawlers_blocked_count":1,"sitemap_urls":["https://www.kbb.com/sitemap-index.xml","https://www.kbb.com/sitemap-classifieds_condition_make_model_trim.xml.gz","https://www.kbb.com/sitemap-classifieds_condition_make_model_year.xml.gz"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.kbb.com/reviews/pricing-your-next-ford-f-150-it-could-cost-60000-or-more/","status":200,"json_ld_types":["Article","WebPage","ImageObject","BreadcrumbList","WebSite","Organization","Person"],"skipped":null,"sitemap":"https://www.kbb.com/post-sitemap1.xml"}},"performance":{"response_ms":8934,"bytes":400000,"redirects":0,"cookies_set":17},"why_this_score":[{"label":"Akamai detected","points":10},{"label":"Slow response (8934 ms)","points":5},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"strong","signals":[{"kind":"robots_txt","label":"robots.txt blocks 1 AI crawler"},{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/kbb.com.json"},{"schema":"ratradar.site/v1","host":"www.mobile.de","name":"mobile.de","category":"auto","country":"Germany","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Akamai"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"AkamaiGHost"},"robots":{"found":true,"disallow_all":false,"disallow_rules":41,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":59,"bytes":7704,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/mobile.de.json"},{"schema":"ratradar.site/v1","host":"www.autoscout24.com","name":"AutoScout24","category":"auto","country":"Germany","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","A browser load called JSON endpoints that look like data, for example listing-search.api.autoscout24.com/graphql. Untested, check the terms before using them."],"risks":["robots.txt forbids 5 of 14 known AI crawlers (GPTBot, ClaudeBot, CCBot, Google-Extended, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":36,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ClaudeBot","CCBot","Google-Extended","Applebot-Extended"],"ai_crawlers_blocked_count":5,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["listing-search.api.autoscout24.com/graphql"],"inner_page":null},"performance":{"response_ms":747,"bytes":400000,"redirects":0,"cookies_set":4},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 5 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/autoscout24.com.json"},{"schema":"ratradar.site/v1","host":"www.tesla.com","name":"Tesla","category":"auto","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Akamai"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"AkamaiGHost"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":50,"bytes":365,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/tesla.com.json"},{"schema":"ratradar.site/v1","host":"www.ford.com","name":"Ford","category":"auto","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","robots.txt asks for a crawl delay of 10 seconds.","Enumerate pages from the sitemap: https://www.ford.com/sitemap.xml"],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai","Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":19,"crawl_delay_seconds":10,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.ford.com/sitemap.xml","https://www.ford.com/href-sitemap-en-us.xml","https://www.ford.com/accessories/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.ford.com/support/category/motorcraft/products/omnicraft/","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.ford.com/supportsitemap.xml"}},"performance":{"response_ms":10138,"bytes":400000,"redirects":0,"cookies_set":6},"why_this_score":[{"label":"Akamai detected","points":10},{"label":"Fastly detected","points":2},{"label":"Slow response (10138 ms)","points":5},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/ford.com.json"},{"schema":"ratradar.site/v1","host":"crates.io","name":"crates.io","category":"dev","country":"United States","verdict":"open","hostility_score":17,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls.","A browser load called JSON endpoints that look like data, for example crates.io/api/v1/site_metadata. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly"],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":"Heroku"},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["crates.io/api/v1/site_metadata","crates.io/api/v1/summary","play.rust-lang.org/meta/crates"],"inner_page":null},"performance":{"response_ms":97,"bytes":5399,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15},{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/crates.io.json"},{"schema":"ratradar.site/v1","host":"rubygems.org","name":"RubyGems","category":"dev","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","The site publishes 1 RSS or Atom feed. Prefer them for new items."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":"RubyGems.org"},"robots":{"found":true,"disallow_all":false,"disallow_rules":5,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":1,"feed_urls":["https://feeds.feedburner.com/gemcutter-latest"],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":333,"bytes":35397,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/rubygems.org.json"},{"schema":"ratradar.site/v1","host":"central.sonatype.com","name":"Maven Central","category":"dev","country":"United States","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","A browser load called JSON endpoints that look like data, for example central.sonatype.com/api/config. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["central.sonatype.com/api/config","ekr.zdassets.com/compose/67cfc48f-e344-43ae-9fe7-46f37b7eaef6","sonatype5154.zendesk.com/embeddable/config"],"inner_page":null},"performance":{"response_ms":313,"bytes":83086,"redirects":0,"cookies_set":0},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/central.sonatype.com.json"},{"schema":"ratradar.site/v1","host":"pkg.go.dev","name":"pkg.go.dev","category":"dev","country":"United States","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://pkg.go.dev/sitemap/index.xml"],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"Google Frontend"},"robots":{"found":true,"disallow_all":false,"disallow_rules":2,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://pkg.go.dev/sitemap/index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://pkg.go.dev/0xacab.org/warlordsam/vpn-hole/app/alter","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://pkg.go.dev/sitemap/0.xml"}},"performance":{"response_ms":132,"bytes":34115,"redirects":0,"cookies_set":0},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/pkg.go.dev.json"},{"schema":"ratradar.site/v1","host":"packagist.org","name":"Packagist","category":"dev","country":"Germany","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","The site publishes 2 RSS or Atom feeds. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example m58222sh95-dsn.algolia.net/1/indexes/*/queries. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"BunnyCDN-LA1-995"},"robots":{"found":true,"disallow_all":false,"disallow_rules":3,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":2,"feed_urls":["https://packagist.org/feeds/packages.rss","https://packagist.org/feeds/releases.rss"],"api_endpoints_seen":["m58222sh95-dsn.algolia.net/1/indexes/*/queries"],"inner_page":null},"performance":{"response_ms":300,"bytes":18031,"redirects":0,"cookies_set":0},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/packagist.org.json"},{"schema":"ratradar.site/v1","host":"www.nuget.org","name":"NuGet","category":"dev","country":"United States","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","A browser load called JSON endpoints that look like data, for example www.nuget.org/stats/totals. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":2,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.nuget.org/stats/totals"],"inner_page":null},"performance":{"response_ms":239,"bytes":16846,"redirects":0,"cookies_set":2},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/nuget.org.json"},{"schema":"ratradar.site/v1","host":"docs.docker.com","name":"Docker Docs","category":"dev","country":"United States","verdict":"open","hostility_score":8,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://docs.docker.com/sitemap.xml","Read the JSON-LD on the page (TechArticle). It is cleaner than the markup."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront","Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"AmazonS3"},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://docs.docker.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["TechArticle"],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://docs.docker.com/accounts/company/new-company/","status":200,"json_ld_types":["TechArticle","BreadcrumbList"],"skipped":null,"sitemap":"https://docs.docker.com/sitemap.xml"}},"performance":{"response_ms":80,"bytes":185401,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"CloudFront detected","points":2},{"label":"Cloudflare seen by the browser","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/docs.docker.com.json"},{"schema":"ratradar.site/v1","host":"kubernetes.io","name":"Kubernetes Docs","category":"dev","country":"United States","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://kubernetes.io/sitemap.xml","Read the JSON-LD on the page (Organization). It is cleaner than the markup."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Netlify","server":"Netlify"},"robots":{"found":true,"disallow_all":false,"disallow_rules":5,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://kubernetes.io/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://kubernetes.io/docs/reference/kubernetes-api/apps/controller-revision-v1/","status":200,"json_ld_types":["Organization"],"skipped":null,"sitemap":"https://kubernetes.io/en/sitemap.xml"}},"performance":{"response_ms":81,"bytes":41608,"redirects":0,"cookies_set":0},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/kubernetes.io.json"},{"schema":"ratradar.site/v1","host":"docs.python.org","name":"Python Docs","category":"dev","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://docs.python.org/sitemap.xml"],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":22,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://docs.python.org/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://docs.python.org/3.11/","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://docs.python.org/sitemap.xml"}},"performance":{"response_ms":43,"bytes":19625,"redirects":1,"cookies_set":0},"why_this_score":[{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/docs.python.org.json"},{"schema":"ratradar.site/v1","host":"nodejs.org","name":"Node.js","category":"dev","country":"United States","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://nodejs.org/sitemap.xml","The site publishes 1 RSS or Atom feed. Prefer them for new items."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":2,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://nodejs.org/sitemap.xml","https://nodejs.org/learn/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":[],"feeds":1,"feed_urls":["https://nodejs.org/en/feed/blog.xml"],"api_endpoints_seen":[],"inner_page":{"url":"https://nodejs.org/en/blog/video/bert-belder-libuv-lxjs-2012","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://nodejs.org/sitemap.xml"}},"performance":{"response_ms":160,"bytes":400000,"redirects":1,"cookies_set":0},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/nodejs.org.json"},{"schema":"ratradar.site/v1","host":"react.dev","name":"React","category":"dev","country":"United States","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","The site publishes 1 RSS or Atom feed. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example react.dev/_next/data/GqElElY447YALnqZIMkuo/index.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Vercel","server":"Vercel"},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":true,"json_ld_types":[],"feeds":1,"feed_urls":["https://react.dev/rss.xml"],"api_endpoints_seen":["react.dev/_next/data/GqElElY447YALnqZIMkuo/index.json","react.dev/_next/data/GqElElY447YALnqZIMkuo/versions.json","react.dev/_next/data/GqElElY447YALnqZIMkuo/reference/react.json","react.dev/_next/data/GqElElY447YALnqZIMkuo/community.json","react.dev/_next/data/GqElElY447YALnqZIMkuo/community/translations.json","react.dev/_next/data/GqElElY447YALnqZIMkuo/learn.json"],"inner_page":null},"performance":{"response_ms":75,"bytes":272438,"redirects":0,"cookies_set":0},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/react.dev.json"},{"schema":"ratradar.site/v1","host":"www.rust-lang.org","name":"Rust Lang","category":"dev","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":"GitHub.com"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":196,"bytes":18594,"redirects":1,"cookies_set":0},"why_this_score":[{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/rust-lang.org.json"},{"schema":"ratradar.site/v1","host":"go.dev","name":"Go.dev","category":"dev","country":"United States","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","A browser load called JSON endpoints that look like data, for example go.dev/dl/. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"Google Frontend"},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["go.dev/dl/","go.dev/_/fmt"],"inner_page":null},"performance":{"response_ms":119,"bytes":64185,"redirects":0,"cookies_set":0},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/go.dev.json"},{"schema":"ratradar.site/v1","host":"www.typescriptlang.org","name":"TypeScript","category":"dev","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","A browser load called JSON endpoints that look like data, for example www.typescriptlang.org/page-data/index/page-data.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":"GitHub.com"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.typescriptlang.org/page-data/index/page-data.json","www.typescriptlang.org/page-data/app-data.json"],"inner_page":null},"performance":{"response_ms":119,"bytes":258828,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/typescriptlang.org.json"},{"schema":"ratradar.site/v1","host":"www.kernel.org","name":"Linux Kernel","category":"dev","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","The site publishes 2 RSS or Atom feeds. Prefer them for new items."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":"nginx"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":2,"feed_urls":["https://www.kernel.org/feeds/all.atom.xml","https://www.kernel.org/feeds/kdist.xml"],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":23,"bytes":19148,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/kernel.org.json"},{"schema":"ratradar.site/v1","host":"sourceforge.net","name":"SourceForge","category":"dev","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Enumerate pages from the sitemap: https://sourceforge.net/sitemap.xml","Read the JSON-LD on the page (Organization, WebSite). It is cleaner than the markup.","The site publishes 1 RSS or Atom feed. Prefer them for new items."],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":164,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://sourceforge.net/sitemap.xml","https://sourceforge.net/allura_sitemap/sitemap.xml","https://sourceforge.net/directory_sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["Organization","WebSite"],"feeds":1,"feed_urls":["http://sourceforge.net/blog/feed/"],"api_endpoints_seen":[],"inner_page":{"url":"https://sourceforge.net/articles/martec-2019/","status":200,"json_ld_types":["Article","WebPage","ImageObject","BreadcrumbList","WebSite","Person"],"skipped":null,"sitemap":"https://sourceforge.net/articles/post-sitemap.xml"}},"performance":{"response_ms":126,"bytes":124997,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/sourceforge.net.json"},{"schema":"ratradar.site/v1","host":"codeberg.org","name":"Codeberg","category":"dev","country":"Germany","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","robots.txt asks for a crawl delay of 2 seconds."],"risks":["robots.txt forbids 14 of 14 known AI crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":["ratelimit","ratelimit-policy"],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":58,"crawl_delay_seconds":2,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","Claude-Web","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Bytespider","Amazonbot","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":14,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":671,"bytes":34563,"redirects":0,"cookies_set":0},"why_this_score":[],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 14 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/codeberg.org.json"},{"schema":"ratradar.site/v1","host":"hashnode.com","name":"Hashnode","category":"dev","country":"United States","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://hashnode.com/sitemap.xml","Read the JSON-LD on the page (WebSite, Organization, ItemList). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example hashnode.com/api/changelog/recent. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Vercel","server":"Vercel"},"robots":{"found":true,"disallow_all":false,"disallow_rules":11,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://hashnode.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebSite","Organization","ItemList"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["hashnode.com/api/changelog/recent","hashnode.com/api/spotlight"],"inner_page":{"url":"https://hashnode.com/tag/javascript","status":200,"json_ld_types":["WebSite","Organization","BreadcrumbList","CollectionPage"],"skipped":null,"sitemap":"https://hashnode.com/sitemaps/tags.xml"}},"performance":{"response_ms":109,"bytes":379792,"redirects":0,"cookies_set":2},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/hashnode.com.json"},{"schema":"ratradar.site/v1","host":"qiita.com","name":"Qiita","category":"dev","country":"Japan","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Read the JSON-LD on the page (WebSite). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example prg.smartadserver.com/prebid/v1. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":8,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://cdn.qiita.com/sitemap-https/sitemap.xml.gz"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":["WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["prg.smartadserver.com/prebid/v1","ib.adnxs.com/ut/v3/prebid","ad.as.amanad.adtdp.com/v2/prebid","fastlane.rubiconproject.com/a/api/fastlane.json"],"inner_page":null},"performance":{"response_ms":1011,"bytes":317921,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/qiita.com.json"},{"schema":"ratradar.site/v1","host":"zenn.dev","name":"Zenn","category":"dev","country":"Japan","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://zenn.dev/sitemaps/_index.xml","The site publishes 1 RSS or Atom feed. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example zenn.dev/api/me. Untested, check the terms before using them."],"risks":["robots.txt forbids some AI crawlers: Bytespider."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":1,"crawl_delay_seconds":null,"ai_crawlers_blocked":["Bytespider"],"ai_crawlers_blocked_count":1,"sitemap_urls":["https://zenn.dev/sitemaps/_index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":1,"feed_urls":["https://zenn.dev/feed"],"api_endpoints_seen":["zenn.dev/api/me","zenn.dev/api/articles","zenn.dev/api/ribbon_banner","zenn.dev/api/announcement/current"],"inner_page":{"url":"https://zenn.dev/tomokiya/books/4b13342f6d878b93e06c","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://zenn.dev/sitemaps/book1.xml.gz"}},"performance":{"response_ms":393,"bytes":241305,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 1 AI crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/zenn.dev.json"},{"schema":"ratradar.site/v1","host":"www.csdn.net","name":"CSDN","category":"dev","country":"China","verdict":"guarded","hostility_score":35,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429.","A browser load called JSON endpoints that look like data, for example img-home.csdnimg.cn/data_json/toolbar/toolbar1105.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"WAF"},"robots":{"found":true,"disallow_all":false,"disallow_rules":15,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["img-home.csdnimg.cn/data_json/toolbar/toolbar1105.json","kunpeng.csdn.net/ad/template/4606371","kunpeng.csdn.net/ad/template/4606362","kunpeng.csdn.net/ad/template/4562553","kunpeng.csdn.net/ad/template/4514452"],"inner_page":null},"performance":{"response_ms":847,"bytes":2013,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"HTTP 521 on a plain request","points":35}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":521,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/csdn.net.json"},{"schema":"ratradar.site/v1","host":"juejin.cn","name":"Juejin","category":"dev","country":"China","verdict":"open","hostility_score":15,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls.","Enumerate pages from the sitemap: https://juejin.cn/sitemap/posts/index.xml","Read the JSON-LD on the page (WebSite). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example api.juejin.cn/interact_api/v1/pin_tab_lead. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"Tengine"},"robots":{"found":true,"disallow_all":false,"disallow_rules":17,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://juejin.cn/sitemap/posts/index.xml","https://juejin.cn/sitemap/user/index.xml","https://juejin.cn/sitemap/news/index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["api.juejin.cn/interact_api/v1/pin_tab_lead","vcs.zijieapi.com/vc/setting","api.juejin.cn/tag_api/v1/query_category_briefs","api.juejin.cn/user_api/v1/author/recommend","api.juejin.cn/user_api/v1/user/get","mon.zijieapi.com/monitor_web/settings/browser-settings"],"inner_page":{"url":"https://juejin.cn/news/7236934165484142652","status":404,"json_ld_types":[],"skipped":null,"sitemap":"https://juejin.cn/sitemap/news/index1.xml"}},"performance":{"response_ms":620,"bytes":79222,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/juejin.cn.json"},{"schema":"ratradar.site/v1","host":"habr.com","name":"Habr","category":"dev","country":"Russia","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","robots.txt asks for a crawl delay of 10 seconds.","Enumerate pages from the sitemap: https://habr.com/sitemap.xml","A browser load called JSON endpoints that look like data, for example habr.com/effect/a. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"QRATOR"},"robots":{"found":true,"disallow_all":false,"disallow_rules":12,"crawl_delay_seconds":10,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["habr.com/effect/a","habr.com/kek/v2/companies/","habr.com/kek/v2/flows/menu","habr.com/kek/v2/articles/most-reading","yandex.ru/adfox/406261/getBulk/v2"],"inner_page":{"url":"https://habr.com/ru/news/1092822/","status":200,"json_ld_types":["Article"],"skipped":null,"sitemap":"https://habr.com/sitemap_news1.xml"}},"performance":{"response_ms":1443,"bytes":269790,"redirects":1,"cookies_set":1},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/habr.com.json"},{"schema":"ratradar.site/v1","host":"www.freecodecamp.org","name":"freeCodeCamp","category":"edu","country":"United States","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.freecodecamp.org/news/sitemap.xml","A browser load called JSON endpoints that look like data, for example www.freecodecamp.org/page-data/app-data.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.freecodecamp.org/news/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.freecodecamp.org/page-data/app-data.json","www.freecodecamp.org/page-data/index/page-data.json","www.freecodecamp.org/page-data/sq/d/2177391967.json","js.stripe.com/v3/.deploy_status_henson.json","gb.api.freecodecamp.org/api/features/sdk-szxzcLoVPQcu4WH","api.freecodecamp.org/user/session-user"],"inner_page":{"url":"https://www.freecodecamp.org/news/tag/100days100projects/","status":200,"json_ld_types":["Series"],"skipped":null,"sitemap":"https://www.freecodecamp.org/news/sitemap-tags.xml"}},"performance":{"response_ms":252,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/freecodecamp.org.json"},{"schema":"ratradar.site/v1","host":"commons.wikimedia.org","name":"Wikimedia Commons","category":"dev","country":"United States","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Read the JSON-LD on the page (Article). It is cleaner than the markup.","The site publishes 3 RSS or Atom feeds. Prefer them for new items."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"mw-web.codfw.main-6b979c7866-z57bx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":239,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://commons.wikimedia.org/w/rest.php/site/v1/sitemap/0"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":["Article"],"feeds":3,"feed_urls":["https://commons.wikimedia.org/w/api.php?action=featuredfeed&feed=potd&feedformat=atom","https://commons.wikimedia.org/w/api.php?action=featuredfeed&feed=motd&feedformat=atom","https://commons.wikimedia.org/w/index.php?title=Special:RecentChanges&feed=atom"],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":197,"bytes":255130,"redirects":1,"cookies_set":5},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/commons.wikimedia.org.json"},{"schema":"ratradar.site/v1","host":"en.wiktionary.org","name":"Wiktionary","category":"edu","country":"United States","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","The site publishes 3 RSS or Atom feeds. Prefer them for new items."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"mw-web.codfw.main-6b979c7866-ftwb5"},"robots":{"found":true,"disallow_all":false,"disallow_rules":200,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://en.wiktionary.org/w/rest.php/site/v1/sitemap/0"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":3,"feed_urls":["https://en.wiktionary.org/w/api.php?action=featuredfeed&feed=wotd&feedformat=atom","https://en.wiktionary.org/w/api.php?action=featuredfeed&feed=fwotd&feedformat=atom","https://en.wiktionary.org/w/index.php?title=Special:RecentChanges&feed=atom"],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":156,"bytes":170513,"redirects":1,"cookies_set":5},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/en.wiktionary.org.json"},{"schema":"ratradar.site/v1","host":"openlibrary.org","name":"Open Library","category":"edu","country":"United States","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","A browser load called JSON endpoints that look like data, for example sink.archive.org/api/82/envelope/. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"nginx/1.30.5"},"robots":{"found":true,"disallow_all":false,"disallow_rules":16,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://openlibrary.org/static/sitemaps/siteindex.xml.gz"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["sink.archive.org/api/82/envelope/","openlibrary.org/partials/LazyCarousel.json"],"inner_page":null},"performance":{"response_ms":153,"bytes":110352,"redirects":0,"cookies_set":0},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/openlibrary.org.json"},{"schema":"ratradar.site/v1","host":"www.gutenberg.org","name":"Project Gutenberg","category":"edu","country":"United States","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"Apache"},"robots":{"found":true,"disallow_all":false,"disallow_rules":1,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":281,"bytes":30589,"redirects":0,"cookies_set":0},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/gutenberg.org.json"},{"schema":"ratradar.site/v1","host":"www.themoviedb.org","name":"TMDB","category":"media","country":"United States","verdict":"hostile","hostility_score":60,"recommended_approach":"real_browser","steps":["Use a rendering browser, slow requests, and rotating proxies.","A browser load called JSON endpoints that look like data, for example www.themoviedb.org/privacy/7399d694/scripttemplates/202607.1.0/assets/otFlat.json. Untested, check the terms before using them."],"risks":["Hostile to automation. Keep volumes low and cache.","robots.txt forbids 14 of 14 known AI crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":false,"headless_browser":true,"needs_proxies":true,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"openresty"},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","Claude-Web","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Bytespider","Amazonbot","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":14,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.themoviedb.org/privacy/7399d694/scripttemplates/202607.1.0/assets/otFlat.json","www.themoviedb.org/privacy/7399d694/scripttemplates/202607.1.0/assets/v2/otPcTab.json"],"inner_page":null},"performance":{"response_ms":127,"bytes":2616,"redirects":0,"cookies_set":5},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"CloudFront detected","points":2},{"label":"A real browser got in though a plain request did not","points":0}],"eu_ai_training":{"level":"strong","signals":[{"kind":"robots_txt","label":"robots.txt blocks 14 AI crawlers"},{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/themoviedb.org.json"},{"schema":"ratradar.site/v1","host":"musicbrainz.org","name":"MusicBrainz","category":"media","country":"United States","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","robots.txt asks for a crawl delay of 2 seconds."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"Plack::Handler::Starlet"},"robots":{"found":true,"disallow_all":false,"disallow_rules":35,"crawl_delay_seconds":2,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://musicbrainz.org/sitemap-index.xml.gz"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":775,"bytes":35631,"redirects":0,"cookies_set":1},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/musicbrainz.org.json"},{"schema":"ratradar.site/v1","host":"www.last.fm","name":"Last.fm","category":"media","country":"United Kingdom","verdict":"open","hostility_score":8,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.last.fm/sitemap-index-secure.xml","Read the JSON-LD on the page (WebSite). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example castlereach.com/dist/oz8p5c/gk39yjy3tm84vn93b2si8ajfjvm2hovu3un. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly","Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":279,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.last.fm/sitemap-index-secure.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["castlereach.com/dist/oz8p5c/gk39yjy3tm84vn93b2si8ajfjvm2hovu3un","www.last.fm/static/animations/OST-Banner.bbcf5bdc8e9e.json","www.last.fm/static/animations/OST-Find.e4e9ce70199d.json","www.last.fm/static/animations/OST-Rediscover-grey.1e83ab52783b.json"],"inner_page":{"url":"https://www.last.fm/music/Florence+%252B+the+Machine/_/Dog+Days+Are+Over/+lyrics","status":404,"json_ld_types":[],"skipped":null,"sitemap":"https://www.last.fm/sitemap-new-lyrics-secure.xml"}},"performance":{"response_ms":155,"bytes":59880,"redirects":0,"cookies_set":6},"why_this_score":[{"label":"Fastly detected","points":2},{"label":"Cloudflare seen by the browser","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/last.fm.json"},{"schema":"ratradar.site/v1","host":"www.deezer.com","name":"Deezer","category":"media","country":"France","verdict":"open","hostility_score":10,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.deezer.com/sitemap.xml","Read the JSON-LD on the page (Organization, FAQPage). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.deezer.com/MjetNMkGc949P4t1sc2B/3cOihD3razEDNbV3OJ/ORoxZ2Ms/IEYNGj9/xD2A. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"Apache"},"robots":{"found":true,"disallow_all":false,"disallow_rules":18,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.deezer.com/sitemap.xml","https://www.deezer.com/catalog_sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["Organization","FAQPage"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.deezer.com/MjetNMkGc949P4t1sc2B/3cOihD3razEDNbV3OJ/ORoxZ2Ms/IEYNGj9/xD2A","www.deezer.com/ajax/gw-light.php","cdn-files.dzcdn.net/json/iab/vendor-list-en.json"],"inner_page":{"url":"https://www.deezer.com/en/offers/duo/annual-plan","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.deezer.com/offers_sitemap/duo.xml"}},"performance":{"response_ms":482,"bytes":193674,"redirects":1,"cookies_set":4},"why_this_score":[{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/deezer.com.json"},{"schema":"ratradar.site/v1","host":"kick.com","name":"Kick","category":"media","country":"Australia","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://kick.com/sitemap.xml","Read the JSON-LD on the page (Organization, WebSite). It is cleaner than the markup."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":14,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://kick.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["Organization","WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://kick.com/category/slots","status":200,"json_ld_types":["Organization","WebSite"],"skipped":null,"sitemap":"https://kick.com/metadata/categories/sitemap.xml"}},"performance":{"response_ms":170,"bytes":400000,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/kick.com.json"},{"schema":"ratradar.site/v1","host":"rumble.com","name":"Rumble","category":"media","country":"Canada","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":3,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":50,"bytes":5593,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/rumble.com.json"},{"schema":"ratradar.site/v1","host":"rutube.ru","name":"Rutube","category":"media","country":"Russia","verdict":"open","hostility_score":15,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls.","Enumerate pages from the sitemap: https://rutube.ru/sitemap/yandex/index.xml","Read the JSON-LD on the page (WebSite). It is cleaner than the markup.","The site publishes 1 RSS or Atom feed. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example rutube.ru/api/abtests/active-tests/. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":46,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://rutube.ru/sitemap/yandex/index.xml","https://rutube.ru/sitemap/yandex/daily-index.xml","https://rutube.ru/sitemap/yandex/hot-index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebSite"],"feeds":1,"feed_urls":["https://rutube.ru/mrss/video/editorsfeed/"],"api_endpoints_seen":["rutube.ru/api/abtests/active-tests/","rutube.ru/api/page_options/check_vpn","api.expf.ru/api/v1/config.json","rutube.ru/multipass/api/v3/accounts/auth_type/","rutube.ru/api/premier/app/v2.1/onboardings/config/rutube"],"inner_page":null},"performance":{"response_ms":2069,"bytes":400000,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/rutube.ru.json"},{"schema":"ratradar.site/v1","host":"quizlet.com","name":"Quizlet","category":"edu","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans.","robots.txt forbids 8 of 14 known AI crawlers (GPTBot, ClaudeBot, Claude-Web, anthropic-ai, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":48,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ClaudeBot","Claude-Web","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Bytespider"],"ai_crawlers_blocked_count":8,"sitemap_urls":["https://quizlet.com/special-sitemap-index.xml","https://quizlet.com/sitemap_index-notes.xml","https://quizlet.com/special-sitemap-content.xml"]},"machine_readable":{"sitemap":false,"llms_txt":true,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":119,"bytes":160475,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"strong","signals":[{"kind":"robots_txt","label":"robots.txt blocks 8 AI crawlers"},{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/quizlet.com.json"},{"schema":"ratradar.site/v1","host":"www.chegg.com","name":"Chegg","category":"edu","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here."],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"CloudFront"},"robots":{"found":true,"disallow_all":false,"disallow_rules":57,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":197,"bytes":129685,"redirects":0,"cookies_set":3},"why_this_score":[{"label":"CloudFront detected","points":2},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/chegg.com.json"},{"schema":"ratradar.site/v1","host":"www.edx.org","name":"edX","category":"edu","country":"United States","verdict":"open","hostility_score":8,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","robots.txt asks for a crawl delay of 10 seconds.","Enumerate pages from the sitemap: https://www.edx.org/sitemap.xml","Read the JSON-LD on the page (Organization). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.edx.org/api/ecommerce/cart/fetch-cart. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare","CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":23,"crawl_delay_seconds":10,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.edx.org/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.edx.org/api/ecommerce/cart/fetch-cart","www.edx.org/api/feature-flags","www.edx.org/beam/api/v2/sources/bdac7c16-924e-4180-bb35-5b2dfff6d308/evaluations"],"inner_page":{"url":"https://www.edx.org/es/es/aprende/facturacion","status":200,"json_ld_types":["Organization","Article"],"skipped":null,"sitemap":"https://www.edx.org/sitemap.xml"}},"performance":{"response_ms":159,"bytes":400000,"redirects":0,"cookies_set":3},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/edx.org.json"},{"schema":"ratradar.site/v1","host":"ocw.mit.edu","name":"MIT OpenCourseWare","category":"edu","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://ocw.mit.edu/sitemap.xml","Read the JSON-LD on the page (WebPage). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example ph.ol.mit.edu/flags/. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://ocw.mit.edu/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebPage"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["ph.ol.mit.edu/flags/","api.appzi.io/api/probe/Tgs1d"],"inner_page":{"url":"https://ocw.mit.edu/courses/24-231-ethics-fall-2009/resources/24-231f09-th/","status":200,"json_ld_types":["WebPage"],"skipped":null,"sitemap":"https://ocw.mit.edu/courses/24-231-ethics-fall-2009/sitemap.xml"}},"performance":{"response_ms":67,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/ocw.mit.edu.json"},{"schema":"ratradar.site/v1","host":"www.duolingo.com","name":"Duolingo","category":"edu","country":"United States","verdict":"open","hostility_score":17,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls.","Enumerate pages from the sitemap: https://www.duolingo.com/sitemap_index.xml","A browser load called JSON endpoints that look like data, for example d35aaqx5ub95lt.cloudfront.net/lottie/splash/50bda50231c0bce1584e982cebfe8f33.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"duo-api"},"robots":{"found":true,"disallow_all":false,"disallow_rules":25,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.duolingo.com/sitemap_index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["d35aaqx5ub95lt.cloudfront.net/lottie/splash/50bda50231c0bce1584e982cebfe8f33.json","d35aaqx5ub95lt.cloudfront.net/lottie/splash/71b01cd301b53ec9879f2d06eb85f5db.json","d35aaqx5ub95lt.cloudfront.net/lottie/splash/aea5aff1143a9410b81448245ad7c839.json","d35aaqx5ub95lt.cloudfront.net/lottie/splash/82f26795696242931a7b905b4918eb1e.json","d35aaqx5ub95lt.cloudfront.net/lottie/splash/e97b1cde32a58c629a0193eea36ddaab.json"],"inner_page":{"url":"https://www.duolingo.com/course/ar/en/Learn-Arabic","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.duolingo.com/sitemap_courses.xml"}},"performance":{"response_ms":110,"bytes":11291,"redirects":0,"cookies_set":3},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/duolingo.com.json"},{"schema":"ratradar.site/v1","host":"www.skillshare.com","name":"Skillshare","category":"edu","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":11,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.skillshare.com/en/sitemap.xml","https://www.skillshare.com/es/sitemap.xml","https://www.skillshare.com/fr/sitemap.xml"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":27,"bytes":5601,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/skillshare.com.json"},{"schema":"ratradar.site/v1","host":"www.overstock.com","name":"Overstock","category":"shop","country":"United States","verdict":"open","hostility_score":10,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://api.overstock.com/sitemaps/overstock-v3/us/sitemap.xml"],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"Vercel"},"robots":{"found":true,"disallow_all":false,"disallow_rules":25,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://api.overstock.com/sitemaps/overstock-v3/us/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.overstock.com/Jewelry-Watches/DaVonna-14k-Gold-White-Akoya-Pearl-High-Luster-18-inch-Necklace-6.5-7-mm/38998/product.html","status":403,"json_ld_types":[],"skipped":null,"sitemap":"https://api.overstock.com/sitemaps/overstock-v3/us/products.xml"}},"performance":{"response_ms":601,"bytes":369406,"redirects":0,"cookies_set":6},"why_this_score":[{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/overstock.com.json"},{"schema":"ratradar.site/v1","host":"www.williams-sonoma.com","name":"Williams Sonoma","category":"shop","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Enumerate pages from the sitemap: https://www.williams-sonoma.com/netstorage/sitemaps/ecm-sitemap-index.xml"],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":17,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.williams-sonoma.com/netstorage/sitemaps/ecm-sitemap-index.xml","https://www.williams-sonoma.com/netstorage/sitemaps/facet-sitemap-index.xml","https://www.williams-sonoma.com/netstorage/sitemaps/shop-sitemap-index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.williams-sonoma.com/recipe/roasted-pork-with-spicy-pickled-onions.html","status":200,"json_ld_types":["Recipe"],"skipped":null,"sitemap":"https://www.williams-sonoma.com/netstorage/sitemaps/recipe-sitemap-1.xml.gz"}},"performance":{"response_ms":502,"bytes":400000,"redirects":0,"cookies_set":10},"why_this_score":[{"label":"Akamai detected","points":10},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/williams-sonoma.com.json"},{"schema":"ratradar.site/v1","host":"www.gap.com","name":"Gap","category":"shop","country":"United States","verdict":"open","hostility_score":10,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.gap.com/native-sitemap.xml","Read the JSON-LD on the page (Organization, WebSite). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.gap.com/RfPkm/JGdKL/4S/CbqG/ujWA/J7XO2Xp3YauEhL/STdCAQ/DwwFKG/IiYTA. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":16,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.gap.com/native-sitemap.xml","https://www.gap.com/shop/native-custom-sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["Organization","WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.gap.com/RfPkm/JGdKL/4S/CbqG/ujWA/J7XO2Xp3YauEhL/STdCAQ/DwwFKG/IiYTA","www.gap.com/resources/personalization/v1/0DF568DD1F5D4E6CA0710B91C6A5B341","www.gap.com/RfPkm/JGdKL/4S/CbqG/ujWA/fkXO2Xp3/MkJJAQ/QG1oOm/M6KXkU"],"inner_page":{"url":"https://www.gap.com/browse/women/maternity?cid=1127956&color=clear","status":200,"json_ld_types":["BreadcrumbList"],"skipped":null,"sitemap":"https://www.gap.com/native-color-sitemap.xml"}},"performance":{"response_ms":127,"bytes":400000,"redirects":0,"cookies_set":6},"why_this_score":[{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/gap.com.json"},{"schema":"ratradar.site/v1","host":"www.uniqlo.com","name":"Uniqlo","category":"shop","country":"Japan","verdict":"hostile","hostility_score":55,"recommended_approach":"real_browser","steps":["Use a rendering browser, slow requests, and rotating proxies."],"risks":["Hostile to automation. Keep volumes low and cache."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":15040,"bytes":null,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"No answer to a plain request","points":40},{"label":"Akamai seen by the browser","points":10},{"label":"Slow response (15040 ms)","points":5}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":0,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/uniqlo.com.json"},{"schema":"ratradar.site/v1","host":"www.asos.com","name":"ASOS","category":"shop","country":"United Kingdom","verdict":"guarded","hostility_score":45,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":15039,"bytes":null,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"No answer to a plain request","points":40},{"label":"Slow response (15039 ms)","points":5}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":0,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/asos.com.json"},{"schema":"ratradar.site/v1","host":"www.boohoo.com","name":"Boohoo","category":"shop","country":"United Kingdom","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Enumerate pages from the sitemap: https://www.boohoo.com/buy/nc/sitemap/keyword.xml","Read the JSON-LD on the page (WebSite, Organization). It is cleaner than the markup."],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["AWS WAF","CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"CloudFront"},"robots":{"found":true,"disallow_all":false,"disallow_rules":14,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.boohoo.com/buy/nc/sitemap/keyword.xml","https://www.boohoo.com/sitemap/boohooww/sitemap.xml","https://www.boohoo.com/sitemap/boohooww/products-0.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebSite","Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.boohoo.com/product/truclothing-sunwave---suitcase-expandable-hard-shell-4-spinner-wheels_p-9f7ab1af-9818-4876-acf0-777e392190a6","status":200,"json_ld_types":["Product","BreadcrumbList"],"skipped":null,"sitemap":"https://www.boohoo.com/sitemap/boohooww/products-0.xml"}},"performance":{"response_ms":166,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"AWS WAF detected","points":12},{"label":"CloudFront detected","points":2},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/boohoo.com.json"},{"schema":"ratradar.site/v1","host":"www.very.co.uk","name":"Very","category":"shop","country":"United Kingdom","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Akamai"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"AkamaiGHost"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":52,"bytes":4306,"redirects":0,"cookies_set":5},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/very.co.uk.json"},{"schema":"ratradar.site/v1","host":"www.johnlewis.com","name":"John Lewis","category":"shop","country":"United Kingdom","verdict":"open","hostility_score":10,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.johnlewis.com/siteindex.xml","Read the JSON-LD on the page (Organization, WebSite). It is cleaner than the markup."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":34,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.johnlewis.com/siteindex.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["Organization","WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.johnlewis.com/customer-services/shopping-with-us/terms-and-conditions/ocab-terms-conditions","status":200,"json_ld_types":["Organization","WebSite"],"skipped":null,"sitemap":"https://www.johnlewis.com/shops-services.xml"}},"performance":{"response_ms":404,"bytes":400000,"redirects":0,"cookies_set":4},"why_this_score":[{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":0,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/johnlewis.com.json"},{"schema":"ratradar.site/v1","host":"www.currys.co.uk","name":"Currys","category":"shop","country":"United Kingdom","verdict":"blocked","hostility_score":76,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":88,"bytes":385630,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/currys.co.uk.json"},{"schema":"ratradar.site/v1","host":"www.boots.com","name":"Boots","category":"shop","country":"United Kingdom","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","A browser load called JSON endpoints that look like data, for example www.boots.com/deot-thath-I-my-bothinke-you-weywarley-violent-C. Untested, check the terms before using them."],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Imperva"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":31,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.boots.com/sitemap_11352.xml"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.boots.com/deot-thath-I-my-bothinke-you-weywarley-violent-C"],"inner_page":null},"performance":{"response_ms":103,"bytes":6183,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"Imperva detected","points":12},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/boots.com.json"},{"schema":"ratradar.site/v1","host":"www.sainsburys.co.uk","name":"Sainsburys","category":"shop","country":"United Kingdom","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Akamai"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":259,"bytes":376,"redirects":0,"cookies_set":3},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/sainsburys.co.uk.json"},{"schema":"ratradar.site/v1","host":"www.mediamarkt.de","name":"MediaMarkt","category":"shop","country":"Germany","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies.","Enumerate pages from the sitemap: https://www.mediamarkt.de/sitemaps/sitemap-index.xml"],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans.","robots.txt forbids 3 of 14 known AI crawlers (CCBot, Amazonbot, cohere-ai). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":28,"crawl_delay_seconds":null,"ai_crawlers_blocked":["CCBot","Amazonbot","cohere-ai"],"ai_crawlers_blocked_count":3,"sitemap_urls":["https://www.mediamarkt.de/sitemaps/sitemap-index.xml","https://www.mediamarkt.de/sitemaps/magazin-deals-angebote.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":545,"bytes":13266,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"strong","signals":[{"kind":"robots_txt","label":"robots.txt blocks 3 AI crawlers"},{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/mediamarkt.de.json"},{"schema":"ratradar.site/v1","host":"www.fnac.com","name":"Fnac","category":"shop","country":"France","verdict":"blocked","hostility_score":80,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":""},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":453,"bytes":70158,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/fnac.com.json"},{"schema":"ratradar.site/v1","host":"www.bol.com","name":"Bol.com","category":"shop","country":"Netherlands","verdict":"blocked","hostility_score":80,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies.","Enumerate pages from the sitemap: https://www.bol.com/sitemap/nl-nl/"],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"AkamaiNetStorage"},"robots":{"found":true,"disallow_all":false,"disallow_rules":55,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.bol.com/sitemap/nl-nl/","https://www.bol.com/sitemap/nl-be/","https://www.bol.com/sitemap/fr-be/"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":54,"bytes":12569,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/bol.com.json"},{"schema":"ratradar.site/v1","host":"www.coolblue.nl","name":"Coolblue","category":"shop","country":"Netherlands","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Enumerate pages from the sitemap: https://sitemaps.coolblue.nl/www.coolblue.nl/nl/products_1.xml","Read the JSON-LD on the page (WebSite). It is cleaner than the markup."],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":313,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://sitemaps.coolblue.nl/www.coolblue.nl/nl/products_1.xml","https://sitemaps.coolblue.nl/www.coolblue.nl/en/products_1.xml","https://sitemaps.coolblue.nl/www.coolblue.nl/custom.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.coolblue.nl/en/vacancies/team-lead-customer-service-german-speaking","status":200,"json_ld_types":["JobPosting"],"skipped":null,"sitemap":"https://www.coolblue.nl/en/vacancies/sitemap.xml"}},"performance":{"response_ms":373,"bytes":400000,"redirects":0,"cookies_set":5},"why_this_score":[{"label":"CloudFront detected","points":2},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/coolblue.nl.json"},{"schema":"ratradar.site/v1","host":"www.elkjop.no","name":"Elkjop","category":"shop","country":"Norway","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","A browser load called JSON endpoints that look like data, for example www.elkjop.no/api/algolia/signed-api-key. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Vercel","server":"Vercel"},"robots":{"found":true,"disallow_all":false,"disallow_rules":49,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.elkjop.no/sitemaps/OCNOELK.pdp.index.sitemap.xml","https://www.elkjop.no/service-sitemap-site-elkjop-no-no-sitemap_index.xml"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.elkjop.no/api/algolia/signed-api-key"],"inner_page":null},"performance":{"response_ms":613,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/elkjop.no.json"},{"schema":"ratradar.site/v1","host":"www.elgiganten.se","name":"Elgiganten","category":"shop","country":"Sweden","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","A browser load called JSON endpoints that look like data, for example www.elgiganten.se/api/algolia/signed-api-key. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Vercel","server":"Vercel"},"robots":{"found":true,"disallow_all":false,"disallow_rules":49,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.elgiganten.se/sitemaps/OCSEELG.pdp.index.sitemap.xml","https://www.elgiganten.se/service-sitemap-site-elgiganten-se-sv-sitemap_index.xml"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.elgiganten.se/api/algolia/signed-api-key","www.elgiganten.se/api/trpc/product.getStockVisibility"],"inner_page":null},"performance":{"response_ms":998,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/elgiganten.se.json"},{"schema":"ratradar.site/v1","host":"www.pccomponentes.com","name":"PcComponentes","category":"shop","country":"Spain","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":37,"bytes":5590,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/pccomponentes.com.json"},{"schema":"ratradar.site/v1","host":"www.elcorteingles.es","name":"El Corte Ingles","category":"shop","country":"Spain","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Akamai"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"AkamaiGHost"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":73,"bytes":374,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/elcorteingles.es.json"},{"schema":"ratradar.site/v1","host":"www.magazineluiza.com.br","name":"Magalu","category":"shop","country":"Brazil","verdict":"blocked","hostility_score":80,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":123,"bytes":1082,"redirects":0,"cookies_set":29},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/magazineluiza.com.br.json"},{"schema":"ratradar.site/v1","host":"www.americanas.com.br","name":"Americanas","category":"shop","country":"Brazil","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.americanas.com.br/sitemap.xml","Read the JSON-LD on the page (WebSite, Organization). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.americanas.com.br/api/graphql. Untested, check the terms before using them."],"risks":["robots.txt forbids 3 of 14 known AI crawlers (GPTBot, CCBot, Google-Extended). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"CloudFront"},"robots":{"found":true,"disallow_all":false,"disallow_rules":12,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","CCBot","Google-Extended"],"ai_crawlers_blocked_count":3,"sitemap_urls":["https://www.americanas.com.br/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["WebSite","Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.americanas.com.br/api/graphql","www.americanas.com.br/api/sessions","reviews-api.konfidency.com.br/americanas/info","cdn.newtail.com.br/retail-media/configurations/americanas.com.br.json","ucd-web.api.useinsider.com/v1/get"],"inner_page":{"url":"https://www.americanas.com.br/forno-eletrico-mueller-44-litros-sonetto-stile-g3-preto-127-volts-7390937631/p","status":200,"json_ld_types":["BreadcrumbList","Product"],"skipped":null,"sitemap":"https://www.americanas.com.br/sitemap/product-0.xml"}},"performance":{"response_ms":55,"bytes":247765,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 3 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/americanas.com.br.json"},{"schema":"ratradar.site/v1","host":"www.falabella.com","name":"Falabella","category":"shop","country":"Chile","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.falabella.com/static/site/sitemaps/collections/collections_cl_FA_COM-index.xml","Read the JSON-LD on the page (WebSite, Organization). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.falabella.com/s/browse/v1/fetchItemDetails/cl. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":5,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.falabella.com/static/site/sitemaps/collections/collections_cl_FA_COM-index.xml","https://www.falabella.com/static/site/sitemaps/landings/landings_cl_FA_COM-index.xml","https://www.falabella.com/static/site/sitemaps/brands/brands_cl_FA_COM-index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebSite","Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.falabella.com/s/browse/v1/fetchItemDetails/cl","www.falabella.com/s/commerce-config/v1/FACL/commerce-config/merged","www.falabella.com/s/geo/v2/districts/cl"],"inner_page":{"url":"https://www.falabella.com/falabella-cl/product/144811750/Velador-2-Niveles-Con-Cajones-Homer-Design-40*30*53/144811751","status":403,"json_ld_types":[],"skipped":null,"sitemap":"https://www.falabella.com/static/site/sitemaps/pdp/pdp_cl_FA_COM-0.xml"}},"performance":{"response_ms":123,"bytes":400000,"redirects":1,"cookies_set":8},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/falabella.com.json"},{"schema":"ratradar.site/v1","host":"www.liverpool.com.mx","name":"Liverpool MX","category":"shop","country":"Mexico","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Akamai"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"AkamaiGHost"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":35,"bytes":376,"redirects":0,"cookies_set":4},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/liverpool.com.mx.json"},{"schema":"ratradar.site/v1","host":"www.walmart.com.mx","name":"Walmart Mexico","category":"shop","country":"Mexico","verdict":"guarded","hostility_score":30,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429.","Enumerate pages from the sitemap: https://www.walmart.com.mx/siteindex.xml","Read the JSON-LD on the page (Organization, WebSite). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.walmart.com.mx/TfHzwS/YOHa/J0bPh/gED/xD0rhuyrs/i77p4SrcbpGJNVipiw/fnRjRX5JAQ/XD/BrPU50G2U. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai","PerimeterX"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":8,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.walmart.com.mx/siteindex.xml","https://www.walmart.com.mx/contentSitemapAdhoc.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["Organization","WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.walmart.com.mx/TfHzwS/YOHa/J0bPh/gED/xD0rhuyrs/i77p4SrcbpGJNVipiw/fnRjRX5JAQ/XD/BrPU50G2U"],"inner_page":{"url":"https://www.walmart.com.mx/ip/1-juego-de-tabla-de-madera-para-noquis-varilla-para-enrollar-en-casa-palo-para-enrollar-en-la-cocina/00368397307819","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.walmart.com.mx/productSitemap.xml"}},"performance":{"response_ms":1266,"bytes":400000,"redirects":0,"cookies_set":33},"why_this_score":[{"label":"Akamai detected","points":10},{"label":"PerimeterX detected","points":20}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/walmart.com.mx.json"},{"schema":"ratradar.site/v1","host":"www.jd.com","name":"JD.com","category":"shop","country":"China","verdict":"open","hostility_score":15,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls.","A browser load called JSON endpoints that look like data, for example jdcorporateblog.com/wp-json/news/list. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"jfe"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["jdcorporateblog.com/wp-json/news/list"],"inner_page":null},"performance":{"response_ms":1040,"bytes":686,"redirects":1,"cookies_set":0},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/jd.com.json"},{"schema":"ratradar.site/v1","host":"www.pinduoduo.com","name":"Pinduoduo","category":"shop","country":"China","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","A browser load called JSON endpoints that look like data, for example www.pinduoduo.com/icp. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.pinduoduo.com/icp","apiv2.pinduoduo.com/api/server/_stm","apiv2.pinduoduo.com/api/gindex/tf/query_tf_goods_info"],"inner_page":null},"performance":{"response_ms":725,"bytes":12049,"redirects":0,"cookies_set":0},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/pinduoduo.com.json"},{"schema":"ratradar.site/v1","host":"www.tmall.com","name":"Tmall","category":"shop","country":"China","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.tmall.com/sitemap.xml","A browser load called JSON endpoints that look like data, for example umdcv4.tmall.com/repTw.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"Tengine"},"robots":{"found":true,"disallow_all":false,"disallow_rules":1,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["umdcv4.tmall.com/repTw.json","umdc.tmall.com/wcfg.json","h5api.m.tmall.com/h5/mtop.tmall.kangaroo.core.service.route.aldlampservicefixedresv2/1.0/","h5api.m.tmall.com/h5/mtop.user.getusersimple/1.0/","h5api.m.tmall.com/h5/mtop.alibaba.fc.api.maoxland.containerfacade.singleview/1.0/","h5api.m.tmall.com/h5/mtop.relationrecommend.wirelessrecommend.recommend/2.0/"],"inner_page":null},"performance":{"response_ms":411,"bytes":108760,"redirects":0,"cookies_set":0},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/tmall.com.json"},{"schema":"ratradar.site/v1","host":"www.qoo10.sg","name":"Qoo10","category":"market","country":"Singapore","verdict":"guarded","hostility_score":40,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":57,"bytes":null,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"No answer to a plain request","points":40}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":0,"headless_browser_status":0,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/qoo10.sg.json"},{"schema":"ratradar.site/v1","host":"www.gmarket.co.kr","name":"Gmarket","category":"shop","country":"South Korea","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans.","robots.txt forbids 11 of 14 known AI crawlers (GPTBot, ClaudeBot, Claude-Web, anthropic-ai, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ClaudeBot","Claude-Web","anthropic-ai","CCBot","Google-Extended","Bytespider","Amazonbot","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":11,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":56,"bytes":17635,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"strong","signals":[{"kind":"robots_txt","label":"robots.txt blocks 11 AI crawlers"},{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/gmarket.co.kr.json"},{"schema":"ratradar.site/v1","host":"www.11st.co.kr","name":"11st","category":"shop","country":"South Korea","verdict":"hostile","hostility_score":50,"recommended_approach":"respect_robots_skip","steps":["robots.txt disallows every crawler. Do not scrape it. Look for an official API, a data license, or a partner feed."],"risks":["Scraping against an explicit robots.txt ban is the quickest way to a legal or blocking problem."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":false,"official_api_advised":true},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":true,"disallow_rules":1,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":442,"bytes":11426,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15},{"label":"robots.txt disallows everything","points":10},{"label":"Raised to at least 50 because robots.txt disallows everything","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt bans every crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/11st.co.kr.json"},{"schema":"ratradar.site/v1","host":"shopping.yahoo.co.jp","name":"Yahoo Shopping JP","category":"shop","country":"Japan","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","A browser load called JSON endpoints that look like data, for example www.googleadservices.com/pagead/conversion/765651919/. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":272,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.googleadservices.com/pagead/conversion/765651919/"],"inner_page":null},"performance":{"response_ms":1315,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Cloudflare seen by the browser","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/shopping.yahoo.co.jp.json"},{"schema":"ratradar.site/v1","host":"www.myntra.com","name":"Myntra","category":"shop","country":"India","verdict":"guarded","hostility_score":45,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":15042,"bytes":null,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"No answer to a plain request","points":40},{"label":"Slow response (15042 ms)","points":5}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":0,"headless_browser_status":0,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/myntra.com.json"},{"schema":"ratradar.site/v1","host":"www.nykaa.com","name":"Nykaa","category":"shop","country":"India","verdict":"guarded","hostility_score":40,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429.","Enumerate pages from the sitemap: https://www.nykaa.com/sitemap-v2/sitemap-index.xml"],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":13,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.nykaa.com/sitemap-v2/sitemap-index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":564,"bytes":null,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"No answer to a plain request","points":40}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":0,"headless_browser_status":0,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/nykaa.com.json"},{"schema":"ratradar.site/v1","host":"www.snapdeal.com","name":"Snapdeal","category":"shop","country":"India","verdict":"hostile","hostility_score":72,"recommended_approach":"real_browser","steps":["Use a rendering browser, slow requests, and rotating proxies."],"risks":["Hostile to automation. Keep volumes low and cache."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"CloudFront"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":637,"bytes":919,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/snapdeal.com.json"},{"schema":"ratradar.site/v1","host":"www.daraz.pk","name":"Daraz","category":"shop","country":"Pakistan","verdict":"open","hostility_score":5,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.daraz.pk/sitemap.xml","A browser load called JSON endpoints that look like data, for example acs-m.daraz.pk/h5/mtop.lazada.member.user.biz.getuser/1.0/. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"Tengine/Aserver"},"robots":{"found":true,"disallow_all":false,"disallow_rules":15,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["acs-m.daraz.pk/h5/mtop.lazada.member.user.biz.getuser/1.0/","acs-m.daraz.pk/h5/mtop.relationrecommend.lazadarecommend.recommend/1.0/","member.daraz.pk/user/api/getUser","member.daraz.pk/user/api/getCsrfToken","my.daraz.pk/api/recentOrders/"],"inner_page":null},"performance":{"response_ms":3771,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Slow response (3771 ms)","points":5}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/daraz.pk.json"},{"schema":"ratradar.site/v1","host":"www.bukalapak.com","name":"Bukalapak","category":"shop","country":"Indonesia","verdict":"guarded","hostility_score":37,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":""},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":235,"bytes":null,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"HTTP 406 on a plain request","points":35},{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":406,"headless_browser_status":0,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/bukalapak.com.json"},{"schema":"ratradar.site/v1","host":"www.konga.com","name":"Konga","category":"shop","country":"Nigeria","verdict":"open","hostility_score":8,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.konga.com/sitemap.xml","Read the JSON-LD on the page (ItemList). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example api.konga.com/v1/graphql. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare","CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":7,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.konga.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["ItemList"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["api.konga.com/v1/graphql"],"inner_page":{"url":"https://www.konga.com/deals/daily","status":200,"json_ld_types":["ItemList"],"skipped":null,"sitemap":"https://www.konga.com/sitemap.xml"}},"performance":{"response_ms":2166,"bytes":400000,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/konga.com.json"},{"schema":"ratradar.site/v1","host":"www.amazon.ae","name":"Amazon UAE","category":"shop","country":"United Arab Emirates","verdict":"hostile","hostility_score":59,"recommended_approach":"real_browser","steps":["Use a rendering browser, slow requests, and rotating proxies.","A browser load called JSON endpoints that look like data, for example www.amazon.ae/dram/renderLazyLoadedStreaming. Untested, check the terms before using them."],"risks":["Hostile to automation. Keep volumes low and cache.","robots.txt forbids 11 of 14 known AI crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":false,"headless_browser":true,"needs_proxies":true,"official_api_advised":false},"protections":{"vendors":["AWS WAF","CloudFront"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"CloudFront"},"robots":{"found":true,"disallow_all":false,"disallow_rules":79,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","Claude-Web","CCBot","PerplexityBot","Google-Extended","Bytespider","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":11,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.amazon.ae/dram/renderLazyLoadedStreaming"],"inner_page":null},"performance":{"response_ms":42,"bytes":2007,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"A challenge page appeared","points":45},{"label":"AWS WAF detected","points":12},{"label":"CloudFront detected","points":2},{"label":"A real browser got in though a plain request did not","points":0}],"eu_ai_training":{"level":"strong","signals":[{"kind":"robots_txt","label":"robots.txt blocks 11 AI crawlers"},{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":202,"headless_browser_status":202,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/amazon.ae.json"},{"schema":"ratradar.site/v1","host":"www.hepsiburada.com","name":"Hepsiburada","category":"shop","country":"Turkey","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Akamai"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"AkamaiGHost"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":127,"bytes":1755,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/hepsiburada.com.json"},{"schema":"ratradar.site/v1","host":"www.trendyol.com","name":"Trendyol","category":"shop","country":"Turkey","verdict":"open","hostility_score":23,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare","CloudFront"],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":566,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":true,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":526,"bytes":84079,"redirects":1,"cookies_set":8},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15},{"label":"Cloudflare detected","points":6},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":0,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/trendyol.com.json"},{"schema":"ratradar.site/v1","host":"www.ozon.ru","name":"Ozon","category":"shop","country":"Russia","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here."],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"nginx"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":20239,"bytes":164,"redirects":37,"cookies_set":1},"why_this_score":[{"label":"Slow response (20239 ms)","points":5},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":307,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/ozon.ru.json"},{"schema":"ratradar.site/v1","host":"www.wildberries.ru","name":"Wildberries","category":"shop","country":"Russia","verdict":"guarded","hostility_score":35,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"wbaas"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":577,"bytes":1136,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"HTTP 498 on a plain request","points":35}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":498,"headless_browser_status":498,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/wildberries.ru.json"},{"schema":"ratradar.site/v1","host":"rozetka.com.ua","name":"Rozetka","category":"shop","country":"Ukraine","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":33,"bytes":5575,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/rozetka.com.ua.json"},{"schema":"ratradar.site/v1","host":"www.emag.ro","name":"eMAG","category":"shop","country":"Romania","verdict":"hostile","hostility_score":74,"recommended_approach":"real_browser","steps":["Use a rendering browser, slow requests, and rotating proxies."],"risks":["Hostile to automation. Keep volumes low and cache."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":false},"protections":{"vendors":["AWS WAF","CloudFront"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"CloudFront"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":204,"bytes":2991,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"HTTP 511 on a plain request","points":35},{"label":"A challenge page appeared","points":25},{"label":"AWS WAF detected","points":12},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":511,"headless_browser_status":410,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/emag.ro.json"},{"schema":"ratradar.site/v1","host":"www.skroutz.gr","name":"Skroutz","category":"shop","country":"Greece","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans.","robots.txt forbids 9 of 14 known AI crawlers (GPTBot, OAI-SearchBot, ClaudeBot, anthropic-ai, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":34,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","OAI-SearchBot","ClaudeBot","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Bytespider","Amazonbot"],"ai_crawlers_blocked_count":9,"sitemap_urls":["https://www.skroutz.gr/sitemaps/sitemap_index.xml.gz"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":34,"bytes":5597,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"strong","signals":[{"kind":"robots_txt","label":"robots.txt blocks 9 AI crawlers"},{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/skroutz.gr.json"},{"schema":"ratradar.site/v1","host":"www.alza.cz","name":"Alza","category":"shop","country":"Czechia","verdict":"blocked","hostility_score":76,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":50,"bytes":65556,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/alza.cz.json"},{"schema":"ratradar.site/v1","host":"www.amazon.co.uk","name":"Amazon UK","category":"shop","country":"United Kingdom","verdict":"hostile","hostility_score":59,"recommended_approach":"real_browser","steps":["Use a rendering browser, slow requests, and rotating proxies."],"risks":["Hostile to automation. Keep volumes low and cache.","robots.txt forbids 11 of 14 known AI crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":false,"headless_browser":true,"needs_proxies":true,"official_api_advised":false},"protections":{"vendors":["AWS WAF","CloudFront"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"CloudFront"},"robots":{"found":true,"disallow_all":false,"disallow_rules":93,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","Claude-Web","CCBot","PerplexityBot","Google-Extended","Bytespider","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":11,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":24,"bytes":2007,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"A challenge page appeared","points":45},{"label":"AWS WAF detected","points":12},{"label":"CloudFront detected","points":2},{"label":"A real browser got in though a plain request did not","points":0}],"eu_ai_training":{"level":"strong","signals":[{"kind":"robots_txt","label":"robots.txt blocks 11 AI crawlers"},{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":202,"headless_browser_status":202,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/amazon.co.uk.json"},{"schema":"ratradar.site/v1","host":"www.amazon.de","name":"Amazon DE","category":"shop","country":"Germany","verdict":"hostile","hostility_score":60,"recommended_approach":"real_browser","steps":["Use a rendering browser, slow requests, and rotating proxies.","A browser load called JSON endpoints that look like data, for example www.amazon.de/dram/renderLazyLoadedStreaming. Untested, check the terms before using them."],"risks":["Hostile to automation. Keep volumes low and cache.","robots.txt forbids 11 of 14 known AI crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":false,"headless_browser":true,"needs_proxies":true,"official_api_advised":false},"protections":{"vendors":["AWS WAF","CloudFront","Akamai"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"CloudFront"},"robots":{"found":true,"disallow_all":false,"disallow_rules":102,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","Claude-Web","CCBot","PerplexityBot","Google-Extended","Bytespider","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":11,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.amazon.de/dram/renderLazyLoadedStreaming","www.amazon.de/hz/rhf","unagi.amazon.de/1/events/com.amazon.csm.csa.prod"],"inner_page":null},"performance":{"response_ms":38,"bytes":2007,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"A challenge page appeared","points":45},{"label":"AWS WAF detected","points":12},{"label":"CloudFront detected","points":2},{"label":"Akamai seen by the browser","points":10},{"label":"A real browser got in though a plain request did not","points":0}],"eu_ai_training":{"level":"strong","signals":[{"kind":"robots_txt","label":"robots.txt blocks 11 AI crawlers"},{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":202,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/amazon.de.json"},{"schema":"ratradar.site/v1","host":"www.amazon.co.jp","name":"Amazon JP","category":"shop","country":"Japan","verdict":"guarded","hostility_score":25,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls.","A browser load called JSON endpoints that look like data, for example www.amazon.co.jp/hokuto/suggestions. Untested, check the terms before using them."],"risks":["robots.txt forbids 11 of 14 known AI crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"AkamaiGHost"},"robots":{"found":true,"disallow_all":false,"disallow_rules":176,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","Claude-Web","CCBot","PerplexityBot","Google-Extended","Bytespider","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":11,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.amazon.co.jp/hokuto/suggestions"],"inner_page":null},"performance":{"response_ms":55,"bytes":2170,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15},{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 11 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":202,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/amazon.co.jp.json"},{"schema":"ratradar.site/v1","host":"www.amazon.in","name":"Amazon IN","category":"shop","country":"India","verdict":"guarded","hostility_score":25,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls.","A browser load called JSON endpoints that look like data, for example www.amazon.in/dram/renderLazyLoadedStreaming. Untested, check the terms before using them."],"risks":["robots.txt forbids 11 of 14 known AI crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"AkamaiGHost"},"robots":{"found":true,"disallow_all":false,"disallow_rules":82,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","Claude-Web","CCBot","PerplexityBot","Google-Extended","Bytespider","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":11,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.amazon.in/dram/renderLazyLoadedStreaming"],"inner_page":null},"performance":{"response_ms":52,"bytes":2162,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15},{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 11 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":202,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/amazon.in.json"},{"schema":"ratradar.site/v1","host":"www.ebay.co.uk","name":"eBay UK","category":"market","country":"United Kingdom","verdict":"blocked","hostility_score":80,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies.","Enumerate pages from the sitemap: https://www.ebay.co.uk/lst/AUCTION-3-index.xml"],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans.","robots.txt forbids 9 of 14 known AI crawlers (GPTBot, ClaudeBot, anthropic-ai, CCBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"AkamaiGHost"},"robots":{"found":true,"disallow_all":false,"disallow_rules":197,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ClaudeBot","anthropic-ai","CCBot","PerplexityBot","Bytespider","Amazonbot","Applebot-Extended","Meta-ExternalAgent"],"ai_crawlers_blocked_count":9,"sitemap_urls":["https://www.ebay.co.uk/lst/AUCTION-3-index.xml","https://www.ebay.co.uk/lst/BROWSE-3-index.xml","https://www.ebay.co.uk/lst/KWSRP-3-index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":66,"bytes":1831,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 9 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/ebay.co.uk.json"},{"schema":"ratradar.site/v1","host":"www.ebay.de","name":"eBay DE","category":"market","country":"Germany","verdict":"blocked","hostility_score":80,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies.","Enumerate pages from the sitemap: https://www.ebay.de/lst/AUCTION-77-index.xml"],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans.","robots.txt forbids 9 of 14 known AI crawlers (GPTBot, ClaudeBot, anthropic-ai, CCBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"AkamaiGHost"},"robots":{"found":true,"disallow_all":false,"disallow_rules":197,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ClaudeBot","anthropic-ai","CCBot","PerplexityBot","Bytespider","Amazonbot","Applebot-Extended","Meta-ExternalAgent"],"ai_crawlers_blocked_count":9,"sitemap_urls":["https://www.ebay.de/lst/AUCTION-77-index.xml","https://www.ebay.de/lst/BROWSE-77-index.xml","https://www.ebay.de/lst/KWSRP-77-index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":67,"bytes":1831,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 9 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/ebay.de.json"},{"schema":"ratradar.site/v1","host":"www.kijiji.ca","name":"Kijiji","category":"market","country":"Canada","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","A browser load called JSON endpoints that look like data, for example ads.pubmatic.com/AdServer/js/pwt/floors/155984/15027/floors.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"Google Frontend"},"robots":{"found":true,"disallow_all":false,"disallow_rules":233,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":true,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["ads.pubmatic.com/AdServer/js/pwt/floors/155984/15027/floors.json","ut.pubmatic.com/geo","www.kijiji.ca/anvil/api","www.google.com/ccm/geo","ib.adnxs.com/ut/v3/prebid"],"inner_page":null},"performance":{"response_ms":569,"bytes":232254,"redirects":0,"cookies_set":4},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/kijiji.ca.json"},{"schema":"ratradar.site/v1","host":"www.trademe.co.nz","name":"Trade Me","category":"market","country":"New Zealand","verdict":"guarded","hostility_score":25,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls.","Enumerate pages from the sitemap: https://www.trademe.co.nz/a/sitemap_index.xml","A browser load called JSON endpoints that look like data, for example api.trademe.co.nz/v1/dynamiccontentfeed/content.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare","Fastly","CloudFront"],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":79,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.trademe.co.nz/a/sitemap_index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["api.trademe.co.nz/v1/dynamiccontentfeed/content.json","editorial-domain.trademe.co.nz/v1/homepage_promotional_content","ib.adnxs.com/ut/v3/prebid","ads.servenobid.com/adreq","prg.smartadserver.com/prebid/v1"],"inner_page":{"url":"https://www.trademe.co.nz/a/jobs/otago/dunedin","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.trademe.co.nz/a/sitemaps/jobs-sitemap.xml"}},"performance":{"response_ms":594,"bytes":118401,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15},{"label":"Cloudflare detected","points":6},{"label":"Fastly detected","points":2},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/trademe.co.nz.json"},{"schema":"ratradar.site/v1","host":"www.marktplaats.nl","name":"Marktplaats","category":"market","country":"Netherlands","verdict":"open","hostility_score":14,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.marktplaats.nl/sitemap/sitemap.xml","Read the JSON-LD on the page (Organization, WebSite). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example ads.pubmatic.com/AdServer/js/pwt/162349/14621/67/config.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["AWS WAF","CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":84,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.marktplaats.nl/sitemap/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["Organization","WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["ads.pubmatic.com/AdServer/js/pwt/162349/14621/67/config.json","ut.pubmatic.com/geo"],"inner_page":{"url":"https://www.marktplaats.nl/v/antiek-en-kunst/kunst-schilderijen-modern/m2452093352-schilderij-olieverf-vrouw-kat-maan-b-75cm-x-h-200cm","status":200,"json_ld_types":["BreadcrumbList","Product","ImageObject"],"skipped":null,"sitemap":"https://www.marktplaats.nl/sitemap/c2c_vip.1.sitemap.xml.gz"}},"performance":{"response_ms":392,"bytes":161767,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"AWS WAF detected","points":12},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/marktplaats.nl.json"},{"schema":"ratradar.site/v1","host":"www.subito.it","name":"Subito","category":"market","country":"Italy","verdict":"hostile","hostility_score":60,"recommended_approach":"real_browser","steps":["Use a rendering browser, slow requests, and rotating proxies.","A browser load called JSON endpoints that look like data, for example assets.subito.it/v2/houston/experiments/subito. Untested, check the terms before using them."],"risks":["Hostile to automation. Keep volumes low and cache."],"can_scrape":{"plain_http":false,"headless_browser":true,"needs_proxies":true,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"AkamaiGHost"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["assets.subito.it/v2/houston/experiments/subito","hades.subito.it/v1/search/categories/2/ad_types/s/filters","hades.subito.it/v1/search/items/hp-recommended","cis.subito.it/api/v1/identify","hades.subito.it/v1/search/items"],"inner_page":null},"performance":{"response_ms":42,"bytes":367,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Akamai detected","points":10},{"label":"A real browser got in though a plain request did not","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/subito.it.json"},{"schema":"ratradar.site/v1","host":"es.wallapop.com","name":"Wallapop","category":"market","country":"Spain","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Read the JSON-LD on the page (Organization). It is cleaner than the markup."],"risks":["Hostile to automation. Keep volumes low and cache.","robots.txt forbids some AI crawlers: CCBot.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":14,"crawl_delay_seconds":null,"ai_crawlers_blocked":["CCBot"],"ai_crawlers_blocked_count":1,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":["Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":47,"bytes":267910,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"CloudFront detected","points":2},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 1 AI crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/es.wallapop.com.json"},{"schema":"ratradar.site/v1","host":"www.blocket.se","name":"Blocket","category":"market","country":"Sweden","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.blocket.se/sitemap.xml","Read the JSON-LD on the page (Organization). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example cmpv2.blocket.se/wrapper/v2/meta-data. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":22,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.blocket.se/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["cmpv2.blocket.se/wrapper/v2/meta-data","www.blocket.se/broadcasts","www.blocket.se/frontpage-layout-v2/podium-resource/global-search/api/suggest","cmpv2.blocket.se/wrapper/v2/messages"],"inner_page":{"url":"https://www.blocket.se/bilguiden/biltester-och-provkorningar/test-bmw-x5-ar-fortfarande-bast-i-klassen-460113","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.blocket.se/om/sitemap-articles.xml"}},"performance":{"response_ms":1240,"bytes":370574,"redirects":0,"cookies_set":0},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/blocket.se.json"},{"schema":"ratradar.site/v1","host":"www.thehindu.com","name":"The Hindu","category":"news","country":"India","verdict":"guarded","hostility_score":26,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429.","Enumerate pages from the sitemap: https://www.thehindu.com/sitemap/googlenews/all/all.xml","The site publishes 1 RSS or Atom feed. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example www.thehindu.com/edge/weathertemp. Untested, check the terms before using them."],"risks":["robots.txt forbids 11 of 14 known AI crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare","PerimeterX"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":96,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","Claude-Web","anthropic-ai","CCBot","PerplexityBot","Bytespider","Amazonbot","Meta-ExternalAgent"],"ai_crawlers_blocked_count":11,"sitemap_urls":["https://www.thehindu.com/sitemap/googlenews/all/all.xml","https://www.thehindu.com/sitemap/update/all.xml","https://www.thehindu.com/sitemap/update/section.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":1,"feed_urls":["https://www.thehindu.com/feeder/default.rss"],"api_endpoints_seen":["www.thehindu.com/edge/weathertemp","api-esp.piano.io/publisher/fusion/lucid/data/hash/CE47F7AE-3589-4659-AE18-E46A9BA35B2F","c2.piano.io/xbuilder/experience/execute","entitlements.jwplayer.com/WjCbyFaYEemhjIYJCsgvBA.json","ut.pubmatic.com/geo"],"inner_page":{"url":"https://www.thehindu.com/news/cities/chennai/chennais-beaches-lit-up-with-bioluminescence-many-times/article71571812.ece","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.thehindu.com/sitemap/googlenews/all/all.xml"}},"performance":{"response_ms":78,"bytes":340360,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"PerimeterX detected","points":20}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 11 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/thehindu.com.json"},{"schema":"ratradar.site/v1","host":"www.hindustantimes.com","name":"Hindustan Times","category":"news","country":"India","verdict":"open","hostility_score":10,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.hindustantimes.com/sitemap/news.xml","Read the JSON-LD on the page (WebPage, NewsMediaOrganization, WebSite, MobileApplication). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.hindustantimes.com/static-content/5m/air-quality-v3/delhi.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.hindustantimes.com/sitemap/news.xml","https://www.hindustantimes.com/sitemap/index.xml","https://www.hindustantimes.com/sitemap/liveblog.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["WebPage","NewsMediaOrganization","WebSite","MobileApplication"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.hindustantimes.com/static-content/5m/air-quality-v3/delhi.json","www.hindustantimes.com/static-content/5m/weather/new-delhi.json","www.hindustantimes.com/static-content/5m/htweather.json","web-banner.ads.aps.amazon-adsystem.com/e/dtb/bid","www.hindustantimes.com/cdp/user/ht","www.hindustantimes.com/cdp/api/v4/user/benefit"],"inner_page":{"url":"https://www.hindustantimes.com/lifestyle/recipe/navratri-2026-special-try-chef-sanjeev-kapoor-s-refreshing-hariyali-chaas-recipe-for-a-flavourful-festive-drink-101791704305858.html","status":200,"json_ld_types":["WebPage","BreadcrumbList","Organization","NewsArticle","ViewAction","SiteNavigationElement"],"skipped":null,"sitemap":"https://www.hindustantimes.com/sitemap/news.xml"}},"performance":{"response_ms":94,"bytes":400000,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/hindustantimes.com.json"},{"schema":"ratradar.site/v1","host":"www.dawn.com","name":"Dawn","category":"news","country":"Pakistan","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Enumerate pages from the sitemap: https://www.dawn.com/feeds/sitemap","The site publishes 1 RSS or Atom feed. Prefer them for new items."],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare","CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":14,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.dawn.com/feeds/sitemap"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":1,"feed_urls":["https://www.dawn.com/feeds/home/"],"api_endpoints_seen":[],"inner_page":{"url":"https://www.dawn.com/news/2036398/turkiye-condemns-houthi-attack-on-riyadh-airport","status":200,"json_ld_types":["Organization","NewsArticle"],"skipped":null,"sitemap":"https://www.dawn.com/feeds/sitemap"}},"performance":{"response_ms":68,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"CloudFront detected","points":2},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/dawn.com.json"},{"schema":"ratradar.site/v1","host":"www.straitstimes.com","name":"Straits Times","category":"news","country":"Singapore","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","robots.txt asks for a crawl delay of 10 seconds.","Enumerate pages from the sitemap: https://www.straitstimes.com/livecoverage.xml","Read the JSON-LD on the page (NewsMediaOrganization, WebSite). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example suid.sphlabs.com/. Untested, check the terms before using them."],"risks":["robots.txt forbids some AI crawlers: Google-Extended."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"AmazonS3"},"robots":{"found":true,"disallow_all":false,"disallow_rules":86,"crawl_delay_seconds":10,"ai_crawlers_blocked":["Google-Extended"],"ai_crawlers_blocked_count":1,"sitemap_urls":["https://www.straitstimes.com/livecoverage.xml","https://www.straitstimes.com/googlenews.xml","https://www.straitstimes.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["NewsMediaOrganization","WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["suid.sphlabs.com/","dev.visualwebsiteoptimizer.com/server-side/v2-settings","mab.chartbeat.com/mab_strategy/headline_testing/get_strategy/","neuron.platform.sphdigital.com.sg/config/default.config.json","id.crwdcntrl.net/id","highway.cablecar.sph.com.sg/"],"inner_page":{"url":"https://www.straitstimes.com/sport/2026-singapore-grand-prix-attracts-301294-fans-second-highest-attendance-on-record?ai-allowed=1","status":200,"json_ld_types":["NewsMediaOrganization","NewsArticle"],"skipped":null,"sitemap":"https://www.straitstimes.com/googlenews.xml"}},"performance":{"response_ms":316,"bytes":319239,"redirects":1,"cookies_set":16},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 1 AI crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/straitstimes.com.json"},{"schema":"ratradar.site/v1","host":"www.japantimes.co.jp","name":"Japan Times","category":"news","country":"Japan","verdict":"open","hostility_score":8,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Read the JSON-LD on the page (Website). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.japantimes.co.jp/ajax/getDisqusDetails/. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare","CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":15,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":["Website"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.japantimes.co.jp/ajax/getDisqusDetails/","connect.facebook.net/app_config/json/211376560532885/"],"inner_page":null},"performance":{"response_ms":189,"bytes":400000,"redirects":0,"cookies_set":3},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/japantimes.co.jp.json"},{"schema":"ratradar.site/v1","host":"www.koreaherald.com","name":"Korea Herald","category":"news","country":"South Korea","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.koreaherald.com/portal/google/sectionsitemap","Read the JSON-LD on the page (WebSite). It is cleaner than the markup.","The site publishes 1 RSS or Atom feed. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example api.climateclock.world/v2/widget/clock.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":5,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.koreaherald.com/portal/google/sectionsitemap","https://www.koreaherald.com/sitemap/latest-articles","https://www.koreaherald.com/sitemap/articles/2026"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebSite"],"feeds":1,"feed_urls":["https://www.koreaherald.com/rss/newsAll"],"api_endpoints_seen":["api.climateclock.world/v2/widget/clock.json","apresolve.spotify.com/","guc3-spclient.spotify.com/gabo-receiver-service/public/v3/events"],"inner_page":{"url":"https://www.koreaherald.com/article/10646966","status":200,"json_ld_types":["NewsArticle"],"skipped":null,"sitemap":"https://www.koreaherald.com/sitemap/articles/2026/20260101"}},"performance":{"response_ms":1062,"bytes":96469,"redirects":0,"cookies_set":2},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/koreaherald.com.json"},{"schema":"ratradar.site/v1","host":"www.smh.com.au","name":"Sydney Morning Herald","category":"news","country":"Australia","verdict":"open","hostility_score":8,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.smh.com.au/sitemaps/news/brands/smh","Read the JSON-LD on the page (WebPage). It is cleaner than the markup.","The site publishes 17 RSS or Atom feeds. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example api.smh.com.au/api/content/v0/assets/p60x8t. Untested, check the terms before using them."],"risks":["robots.txt forbids 14 of 14 known AI crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly","Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":5,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","Claude-Web","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Bytespider","Amazonbot","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":14,"sitemap_urls":["https://www.smh.com.au/sitemaps/news/brands/smh","https://www.smh.com.au/sitemaps/smh-sitemaps-videos.xml","https://www.smh.com.au/sitemaps/smh-sitemaps-articles.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebPage"],"feeds":17,"feed_urls":["https://www.smh.com.au/rss/feed.xml","https://www.smh.com.au/rss/politics/federal.xml","https://www.smh.com.au/rss/national/nsw.xml"],"api_endpoints_seen":["api.smh.com.au/api/content/v0/assets/p60x8t","api.smh.com.au/graphql","api.smh.com.au/api/content/v0/assets/p614ji/posts","news.google.com/swg/_/api/v1/publication/smh.com.au/article","htlb.casalemedia.com/openrtb/pbjs","tlx.3lift.com/header/auction"],"inner_page":{"url":"https://www.smh.com.au/property/news/inner-west-unit-sells-for-981-000-as-market-aligns-with-buyers-20261010-p61479.html","status":200,"json_ld_types":["NewsArticle","BreadcrumbList"],"skipped":null,"sitemap":"https://www.smh.com.au/sitemaps/smh-articles-20261010.xml"}},"performance":{"response_ms":46,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Fastly detected","points":2},{"label":"Cloudflare seen by the browser","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 14 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/smh.com.au.json"},{"schema":"ratradar.site/v1","host":"www.nzherald.co.nz","name":"NZ Herald","category":"news","country":"New Zealand","verdict":"guarded","hostility_score":36,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429.","Enumerate pages from the sitemap: https://www.nzherald.co.nz/arc/outboundfeeds/sitemap-index/?outputType=xml&_website=nzh","A browser load called JSON endpoints that look like data, for example dcinfos-cache.abtasty.com/v1/geoip. Untested, check the terms before using them."],"risks":["robots.txt forbids 14 of 14 known AI crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai","DataDome","Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"openresty"},"robots":{"found":true,"disallow_all":false,"disallow_rules":19,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","Claude-Web","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Bytespider","Amazonbot","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":14,"sitemap_urls":["https://www.nzherald.co.nz/arc/outboundfeeds/sitemap-index/?outputType=xml&_website=nzh","https://www.nzherald.co.nz/arc/outboundfeeds/sitemap-index/?outputType=xml&_website=viva","https://www.nzherald.co.nz/arc/outboundfeeds/sitemap-index/?outputType=xml&_website=bay-of-plenty-times"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["dcinfos-cache.abtasty.com/v1/geoip","dcinfos-cache.abtasty.com/v1/ua-parser","try.abtasty.com/462b232fcb63ec2c862d2638cae0e436/1345516.1667752.json","try.abtasty.com/462b232fcb63ec2c862d2638cae0e436/1438874.1788752.json","www.nzherald.co.nz/pf/api/v3/content/fetch/content-feed-simple"],"inner_page":{"url":"https://www.nzherald.co.nz/rotorua-daily-post/news/owhata-arrest-man-allegedly-pointed-shotgun-at-rotorua-police-officer/MKBEOLADQFBPJGZ2W5RCLAH3N4/","status":200,"json_ld_types":["NewsArticle","BreadcrumbList"],"skipped":null,"sitemap":"https://www.nzherald.co.nz/arc/outboundfeeds/sitemap-news/?outputType=xml&_website=nzh"}},"performance":{"response_ms":263,"bytes":400000,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"Akamai detected","points":10},{"label":"DataDome detected","points":20},{"label":"Cloudflare seen by the browser","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 14 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/nzherald.co.nz.json"},{"schema":"ratradar.site/v1","host":"mg.co.za","name":"Mail and Guardian","category":"news","country":"South Africa","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://mg.co.za/sitemap.xml","Read the JSON-LD on the page (NewsArticle, BreadcrumbList). It is cleaner than the markup.","The site publishes 2 RSS or Atom feeds. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example bridge.newsteam.io/. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":2,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://mg.co.za/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["NewsArticle","BreadcrumbList"],"feeds":2,"feed_urls":["https://mg.co.za/atom/","https://mg.co.za/rss/"],"api_endpoints_seen":["bridge.newsteam.io/","ep1.adtrafficquality.google/getconfig/sodar","bcp.crwdcntrl.net/6/map"],"inner_page":{"url":"https://mg.co.za/news/health/2026-10-09-len-won-t-sell-itself-6-lessons-from-hiv-prevention-pills/","status":200,"json_ld_types":["NewsArticle","BreadcrumbList"],"skipped":null,"sitemap":"https://mg.co.za/sitemap/news/"}},"performance":{"response_ms":115,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/mg.co.za.json"},{"schema":"ratradar.site/v1","host":"nation.africa","name":"Daily Nation","category":"news","country":"Kenya","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":66,"bytes":5616,"redirects":1,"cookies_set":0},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/nation.africa.json"},{"schema":"ratradar.site/v1","host":"english.alarabiya.net","name":"Al Arabiya","category":"news","country":"United Arab Emirates","verdict":"blocked","hostility_score":76,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies.","Enumerate pages from the sitemap: https://english.alarabiya.net/sitemap.xml"],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":2,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://english.alarabiya.net/sitemap.xml","https://english.alarabiya.net/en-images-sitemap.xml","https://english.alarabiya.net/en-videos-sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":53,"bytes":179928,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/english.alarabiya.net.json"},{"schema":"ratradar.site/v1","host":"www.haaretz.com","name":"Haaretz","category":"news","country":"Israel","verdict":"open","hostility_score":8,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.haaretz.com/sitemap.xml","Read the JSON-LD on the page (WebSite, NewsMediaOrganization). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example hb-1055303619817.us-east4.run.app/init. Untested, check the terms before using them."],"risks":["robots.txt forbids 14 of 14 known AI crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly","Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":"istio-envoy"},"robots":{"found":true,"disallow_all":false,"disallow_rules":45,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","Claude-Web","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Bytespider","Amazonbot","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":14,"sitemap_urls":["https://www.haaretz.com/sitemap.xml","https://www.haaretz.com/news-sitemap-content.xml","https://www.haaretz.com/news-sitemap-latest.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebSite","NewsMediaOrganization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["hb-1055303619817.us-east4.run.app/init","lexicon.33across.com/v1/envelope","cdn.valuad.cloud/shaping/haaretzcom.json","cdn.valuad.cloud/floors/haaretzcom.json"],"inner_page":{"url":"https://www.haaretz.com/life/2026-10-11/ty-article-magazine/.premium/the-kids-of-kibbutz-reim-are-the-storytellers-and-stars-of-their-very-own-film/000001a1-29f8-d0ca-a7b7-bffa474a0000","status":200,"json_ld_types":["NewsArticle"],"skipped":null,"sitemap":"https://www.haaretz.com/news-sitemap-latest.xml"}},"performance":{"response_ms":526,"bytes":400000,"redirects":0,"cookies_set":3},"why_this_score":[{"label":"Fastly detected","points":2},{"label":"Cloudflare seen by the browser","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 14 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/haaretz.com.json"},{"schema":"ratradar.site/v1","host":"www.hurriyet.com.tr","name":"Hurriyet","category":"news","country":"Turkey","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Read the JSON-LD on the page (Organization, WebSite). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example pro.ip-api.com/json/. Untested, check the terms before using them."],"risks":["robots.txt forbids 8 of 14 known AI crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"MerlinCDN"},"robots":{"found":true,"disallow_all":false,"disallow_rules":10,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","Claude-Web","anthropic-ai","PerplexityBot","cohere-ai"],"ai_crawlers_blocked_count":8,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":["Organization","WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["pro.ip-api.com/json/","www.hurriyet.com.tr/api/lazyv2/getnotificationcount","cdn.api.btloader.com/device","geoloc.hurriyet.com.tr/api/country/","www.hurriyet.com.tr/api/weather/getweather","wsdkapi.netmera.com/sdk/3.0/config/get"],"inner_page":null},"performance":{"response_ms":1227,"bytes":250763,"redirects":0,"cookies_set":0},"why_this_score":[],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 8 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/hurriyet.com.tr.json"},{"schema":"ratradar.site/v1","host":"www.kommersant.ru","name":"Kommersant","category":"news","country":"Russia","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Read the JSON-LD on the page (WebSite, NewsMediaOrganization). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example yandex.ru/adfox/1092/getBulk/v2. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"QRATOR"},"robots":{"found":true,"disallow_all":false,"disallow_rules":73,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.kommersant.ru/sitemaps/all_docs_sitemap.xml","https://www.kommersant.ru/sitemaps/index_sitemap.xml","https://www.kommersant.ru/sitemaps/news.xml"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":["WebSite","NewsMediaOrganization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["yandex.ru/adfox/1092/getBulk/v2","mc.yandex.com/watch/153166/1","mc.yandex.com/ytm-config/153166","mc.yandex.com/watch/153166"],"inner_page":null},"performance":{"response_ms":1051,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/kommersant.ru.json"},{"schema":"ratradar.site/v1","host":"kyivindependent.com","name":"Kyiv Independent","category":"news","country":"Ukraine","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://kyivindependent.com/news-sitemap.xml","Read the JSON-LD on the page (WebSite, NewsMediaOrganization). It is cleaner than the markup.","The site publishes 1 RSS or Atom feed. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example kyivindependent.com/members/api/member/geo/country. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":12,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://kyivindependent.com/news-sitemap.xml","https://kyivindependent.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["WebSite","NewsMediaOrganization"],"feeds":1,"feed_urls":["https://kyivindependent.com/news-archive/rss/"],"api_endpoints_seen":["kyivindependent.com/members/api/member/geo/country","app.varify.io/v/561","api.smartocto.com/api/brands/tentacles"],"inner_page":{"url":"https://kyivindependent.com/zelensky-says-if-trump-putin-want-elections-in-ukraine-a-two-month-ceasefire-is-needed/","status":200,"json_ld_types":["NewsArticle"],"skipped":null,"sitemap":"https://kyivindependent.com/news-sitemap.xml"}},"performance":{"response_ms":473,"bytes":365820,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/kyivindependent.com.json"},{"schema":"ratradar.site/v1","host":"nos.nl","name":"NOS","category":"news","country":"Netherlands","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://nos.nl/sitemap/index.xml","A browser load called JSON endpoints that look like data, for example nos.nl/api/live-livestreams. Untested, check the terms before using them."],"risks":["robots.txt forbids 11 of 14 known AI crawlers (GPTBot, ChatGPT-User, ClaudeBot, anthropic-ai, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":6,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","ClaudeBot","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Bytespider","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":11,"sitemap_urls":["https://nos.nl/sitemap/index.xml","https://nos.nl/sitemap/news.xml","https://nos.nl/sitemap/livestreams.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["nos.nl/api/live-livestreams","cookies.nos.nl/sites/NOS/nos.nl/version.txt"],"inner_page":{"url":"https://nos.nl/nieuwsuur/artikel/2634404-prominenten-verlaten-nsc-partij-is-volledig-afgedwaald","status":200,"json_ld_types":["WebPage","NewsArticle"],"skipped":null,"sitemap":"https://nos.nl/sitemap/news.xml"}},"performance":{"response_ms":72,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 11 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/nos.nl.json"},{"schema":"ratradar.site/v1","host":"www.nrk.no","name":"NRK","category":"news","country":"Norway","verdict":"open","hostility_score":10,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: http://www.nrk.no/sitemap.xml","The site publishes 1 RSS or Atom feed. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example www.nrk.no/_auth/web/session/tokenforsub. Untested, check the terms before using them."],"risks":["robots.txt forbids 14 of 14 known AI crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":37,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","Claude-Web","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Bytespider","Amazonbot","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":14,"sitemap_urls":["http://www.nrk.no/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":1,"feed_urls":["https://www.nrk.no/toppsaker.rss"],"api_endpoints_seen":["www.nrk.no/_auth/web/session/tokenforsub","pub-apps.nrk.no/nrkno-dketasjen/ditt-distrikt-config","pub-apps.nrk.no/smartpublisering/api/sanity/lanseringer"],"inner_page":{"url":"https://www.nrk.no/video/musikk/bjorn-tomren--under-the-bridge_d147a1db-a2ab-4d34-8c54-d18238b87978","status":200,"json_ld_types":["VideoObject"],"skipped":null,"sitemap":"https://www.nrk.no/sitemap-0.xml"}},"performance":{"response_ms":333,"bytes":400000,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 14 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/nrk.no.json"},{"schema":"ratradar.site/v1","host":"www.svt.se","name":"SVT","category":"news","country":"Sweden","verdict":"open","hostility_score":10,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.svt.se/sitemap.xml","The site publishes 1 RSS or Atom feed. Prefer them for new items."],"risks":["robots.txt forbids 10 of 14 known AI crawlers (GPTBot, ClaudeBot, anthropic-ai, CCBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ClaudeBot","anthropic-ai","CCBot","Google-Extended","Bytespider","Amazonbot","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":10,"sitemap_urls":["https://www.svt.se/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":1,"feed_urls":["https://www.svt.se/rss.xml"],"api_endpoints_seen":[],"inner_page":{"url":"https://www.svt.se/nyheter/video/8117a417c40581e4-se-william-nylanders-mal-i-poangsuccen/h0v4lr","status":200,"json_ld_types":["VideoObject"],"skipped":null,"sitemap":"https://www.svt.se/latest-videos-sitemap.xml"}},"performance":{"response_ms":639,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 10 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/svt.se.json"},{"schema":"ratradar.site/v1","host":"yle.fi","name":"YLE","category":"news","country":"Finland","verdict":"open","hostility_score":12,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Read the JSON-LD on the page (WebSite, SoftwareApplication). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example locations.api.yle.fi/v4/address/current. Untested, check the terms before using them."],"risks":["robots.txt forbids 11 of 14 known AI crawlers (GPTBot, ChatGPT-User, ClaudeBot, Claude-Web, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai","CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":10,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","ClaudeBot","Claude-Web","anthropic-ai","CCBot","Google-Extended","Bytespider","Amazonbot","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":11,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":["WebSite","SoftwareApplication"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["locations.api.yle.fi/v4/address/current","locations.api.yle.fi/v3/address/current","player.api.yle.fi/v1/preview/1-78714949.json"],"inner_page":null},"performance":{"response_ms":722,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Akamai detected","points":10},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 11 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/yle.fi.json"},{"schema":"ratradar.site/v1","host":"www.zeit.de","name":"Die Zeit","category":"news","country":"Germany","verdict":"hostile","hostility_score":72,"recommended_approach":"real_browser","steps":["Use a rendering browser, slow requests, and rotating proxies."],"risks":["Hostile to automation. Keep volumes low and cache.","robots.txt forbids 9 of 14 known AI crawlers (GPTBot, ClaudeBot, anthropic-ai, CCBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":false},"protections":{"vendors":["Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":12,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ClaudeBot","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Bytespider","Applebot-Extended","Meta-ExternalAgent"],"ai_crawlers_blocked_count":9,"sitemap_urls":["https://www.zeit.de/gsitemaps/index.xml"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":124,"bytes":1064,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 9 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/zeit.de.json"},{"schema":"ratradar.site/v1","host":"www.nzz.ch","name":"NZZ","category":"news","country":"Switzerland","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.nzz.ch/sitemap.xml","Read the JSON-LD on the page (NewsMediaOrganization). It is cleaner than the markup."],"risks":["robots.txt forbids 14 of 14 known AI crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":24,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","Claude-Web","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Bytespider","Amazonbot","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":14,"sitemap_urls":["https://www.nzz.ch/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["NewsMediaOrganization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.nzz.ch/video/nzz-format/langfassung","status":200,"json_ld_types":["BreadcrumbList","NewsMediaOrganization"],"skipped":null,"sitemap":"https://www.nzz.ch/sitemap/pages.xml"}},"performance":{"response_ms":230,"bytes":400000,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 14 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/nzz.ch.json"},{"schema":"ratradar.site/v1","host":"www.lefigaro.fr","name":"Le Figaro","category":"news","country":"France","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://sitemaps.lefigaro.fr/lefigaro.fr/articles.xml","Read the JSON-LD on the page (Organization, WebSite, WebPage, ItemList). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.lefigaro.fr/async/breaking_news. Untested, check the terms before using them."],"risks":["robots.txt forbids 10 of 14 known AI crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":10,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","Claude-Web","anthropic-ai","CCBot","Google-Extended","Amazonbot","cohere-ai"],"ai_crawlers_blocked_count":10,"sitemap_urls":["https://sitemaps.lefigaro.fr/lefigaro.fr/articles.xml","https://sitemaps.lefigaro.fr/lefigaro.fr/sections.xml","https://sitemaps.lefigaro.fr/lefigaro.fr/topics.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["Organization","WebSite","WebPage","ItemList"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.lefigaro.fr/async/breaking_news","www.lefigaro.fr/async/news-flash-section/actu","api-graphql.lefigaro.fr/graphql"],"inner_page":{"url":"https://www.lefigaro.fr/story/football-ligue1-2026-2027-6emejournee-paris-sg-vs-le-mans-18809","status":200,"json_ld_types":["Organization","WebSite","WebPage"],"skipped":null,"sitemap":"https://www.lefigaro.fr/sitemap_news.xml"}},"performance":{"response_ms":373,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 10 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/lefigaro.fr.json"},{"schema":"ratradar.site/v1","host":"www.repubblica.it","name":"La Repubblica","category":"news","country":"Italy","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.repubblica.it/sitemap-n.xml","Read the JSON-LD on the page (NewsMediaOrganization). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example cdn.iubenda.com/cs/tcf/v3-versioned/vendorlist.180.json. Untested, check the terms before using them."],"risks":["robots.txt forbids 8 of 14 known AI crawlers (GPTBot, anthropic-ai, CCBot, PerplexityBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":54,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Bytespider","Applebot-Extended","cohere-ai"],"ai_crawlers_blocked_count":8,"sitemap_urls":["https://www.repubblica.it/sitemap-n.xml","https://video.repubblica.it/sitemap-v-day.xml","https://video.repubblica.it/sitemap-v.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["NewsMediaOrganization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["cdn.iubenda.com/cs/tcf/v3-versioned/vendorlist.180.json","cdn.iubenda.com/cs/tcf/gac-vendors.json","connect.facebook.net/app_config/json/182234715127717/"],"inner_page":{"url":"https://www.repubblica.it/sport/calcio/serie-a/2026/10/11/news/fabregas_gasperini_como_roma-425640980/","status":200,"json_ld_types":["NewsArticle","AudioObject","BreadcrumbList"],"skipped":null,"sitemap":"https://www.repubblica.it/sitemap-n.xml"}},"performance":{"response_ms":840,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 8 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/repubblica.it.json"},{"schema":"ratradar.site/v1","host":"www.elmundo.es","name":"El Mundo","category":"news","country":"Spain","verdict":"open","hostility_score":8,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.elmundo.es/sitemaps/directoeditorial_news_a3f3222a2fa5a98b79804fa04675a139.xml","Read the JSON-LD on the page (NewsMediaOrganization). It is cleaner than the markup.","The site publishes 1 RSS or Atom feed. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example adtcdn.unidadeditorial.es/mun/config.json. Untested, check the terms before using them."],"risks":["robots.txt forbids 7 of 14 known AI crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, anthropic-ai, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly","Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":15,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","OAI-SearchBot","anthropic-ai","CCBot","PerplexityBot","Google-Extended"],"ai_crawlers_blocked_count":7,"sitemap_urls":["https://www.elmundo.es/sitemaps/directoeditorial_news_a3f3222a2fa5a98b79804fa04675a139.xml","https://www.elmundo.es/sitemaps/videoct_news_ba306c30e4f41fc8f7a5e2d166e42a3b.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["NewsMediaOrganization"],"feeds":1,"feed_urls":["https://www.elmundo.es/rss/googlenews/portada.xml"],"api_endpoints_seen":["adtcdn.unidadeditorial.es/mun/config.json","www.marca.com/edge-services/user-geo.html","adtcdn.unidadeditorial.es/gexp/getCfg-elmundo-US-d.php","adtcdn2.unidadeditorial.es/gexp/getCfg-elmundo-US-d.php","e00-elmundo.uecdn.es/json/bt/v1/expansion_dmedicina.json","www.elmundo.es/ue-nydus/nydus-feeder.php"],"inner_page":{"url":"https://www.elmundo.es/internacional/2026/10/11/6acb83c2fdddffa1748b458a-video.html","status":200,"json_ld_types":["NewsMediaOrganization","VideoObject","NewsArticle","ReportageNewsArticle","BreadcrumbList"],"skipped":null,"sitemap":"https://www.elmundo.es/sitemaps/videoct_news_ba306c30e4f41fc8f7a5e2d166e42a3b.xml"}},"performance":{"response_ms":241,"bytes":400920,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Fastly detected","points":2},{"label":"Cloudflare seen by the browser","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 7 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/elmundo.es.json"},{"schema":"ratradar.site/v1","host":"www.publico.pt","name":"Publico","category":"news","country":"Portugal","verdict":"hostile","hostility_score":59,"recommended_approach":"real_browser","steps":["Use a rendering browser, slow requests, and rotating proxies."],"risks":["Hostile to automation. Keep volumes low and cache."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":false},"protections":{"vendors":["AWS WAF","CloudFront"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"CloudFront"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":90,"bytes":2016,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"A challenge page appeared","points":45},{"label":"AWS WAF detected","points":12},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":202,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/publico.pt.json"},{"schema":"ratradar.site/v1","host":"wyborcza.pl","name":"Gazeta Wyborcza","category":"news","country":"Poland","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","The site publishes 1 RSS or Atom feed. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example static.im-g.pl/wpc/frontend/wyborcza-podcast-bitbucket/dist/javascript/version-desktop.json. Untested, check the terms before using them."],"risks":["robots.txt forbids 9 of 14 known AI crawlers (GPTBot, ChatGPT-User, ClaudeBot, anthropic-ai, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"Apache"},"robots":{"found":true,"disallow_all":false,"disallow_rules":24,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","ClaudeBot","anthropic-ai","CCBot","PerplexityBot","Bytespider","Applebot-Extended","cohere-ai"],"ai_crawlers_blocked_count":9,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":1,"feed_urls":["https://wyborcza.pl/pub/rss/najnowsze_wyborcza.xml"],"api_endpoints_seen":["static.im-g.pl/wpc/frontend/wyborcza-podcast-bitbucket/dist/javascript/version-desktop.json"],"inner_page":null},"performance":{"response_ms":3000,"bytes":400000,"redirects":1,"cookies_set":5},"why_this_score":[],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 9 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/wyborcza.pl.json"},{"schema":"ratradar.site/v1","host":"www.irishtimes.com","name":"Irish Times","category":"news","country":"Ireland","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.irishtimes.com/arc/outboundfeeds/sitemap/","Read the JSON-LD on the page (WebSite, Organization). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.irishtimes.com/pf/api/v3/content/fetch/story-feed-sections. Untested, check the terms before using them."],"risks":["robots.txt forbids some AI crawlers: GPTBot, ChatGPT-User."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"openresty"},"robots":{"found":true,"disallow_all":false,"disallow_rules":50,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User"],"ai_crawlers_blocked_count":2,"sitemap_urls":["https://www.irishtimes.com/arc/outboundfeeds/sitemap/","https://www.irishtimes.com/arc/outboundfeeds/sitemap-days/","https://www.irishtimes.com/arc/outboundfeeds/sitemap-news-index/"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebSite","Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.irishtimes.com/pf/api/v3/content/fetch/story-feed-sections","www.irishtimes.com/pf/api/v3/content/fetch/story-feed-query","www.irishtimes.com/pf/api/v3/content/fetch/site-service-hierarchy-it","irishtimes-config-prod.api.arc-cdn.net/video/v1/ansvideos/findByUuid"],"inner_page":null},"performance":{"response_ms":104,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 2 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/irishtimes.com.json"},{"schema":"ratradar.site/v1","host":"www.politico.com","name":"Politico","category":"news","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":36,"bytes":5577,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/politico.com.json"},{"schema":"ratradar.site/v1","host":"www.axios.com","name":"Axios","category":"news","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies.","Enumerate pages from the sitemap: https://www.axios.com/sitemap.xml"],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans.","robots.txt forbids 3 of 14 known AI crawlers (CCBot, Bytespider, Amazonbot). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":5,"crawl_delay_seconds":null,"ai_crawlers_blocked":["CCBot","Bytespider","Amazonbot"],"ai_crawlers_blocked_count":3,"sitemap_urls":["https://www.axios.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":59,"bytes":5553,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"strong","signals":[{"kind":"robots_txt","label":"robots.txt blocks 3 AI crawlers"},{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/axios.com.json"},{"schema":"ratradar.site/v1","host":"www.theatlantic.com","name":"The Atlantic","category":"news","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","robots.txt asks for a crawl delay of 1 seconds.","Read the JSON-LD on the page (WebSite, Organization). It is cleaner than the markup.","The site publishes 2 RSS or Atom feeds. Prefer them for new items."],"risks":["Hostile to automation. Keep volumes low and cache.","robots.txt forbids 11 of 14 known AI crawlers (ClaudeBot, Claude-Web, anthropic-ai, CCBot, ...). An agent that identifies as an AI crawler should not fetch this site.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare","Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":9,"crawl_delay_seconds":1,"ai_crawlers_blocked":["ClaudeBot","Claude-Web","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Bytespider","Amazonbot","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":11,"sitemap_urls":["https://www.theatlantic.com/sitemap.xml","https://www.theatlantic.com/sponsored/sitemap.xml"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":["WebSite","Organization"],"feeds":2,"feed_urls":["https://www.theatlantic.com/feed/all/","https://www.theatlantic.com/feed/best-of/"],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":91,"bytes":392148,"redirects":0,"cookies_set":8},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"Fastly detected","points":2},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 11 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/theatlantic.com.json"},{"schema":"ratradar.site/v1","host":"www.newyorker.com","name":"New Yorker","category":"news","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.newyorker.com/sitemap.xml","Read the JSON-LD on the page (Organization, WebSite, ItemList). It is cleaner than the markup.","The site publishes 1 RSS or Atom feed. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example www.newyorker.com/auth/refresh. Untested, check the terms before using them."],"risks":["robots.txt forbids 9 of 14 known AI crawlers (ClaudeBot, CCBot, PerplexityBot, Google-Extended, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"CloudFront"},"robots":{"found":true,"disallow_all":false,"disallow_rules":15,"crawl_delay_seconds":null,"ai_crawlers_blocked":["ClaudeBot","CCBot","PerplexityBot","Google-Extended","Bytespider","Amazonbot","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":9,"sitemap_urls":["https://www.newyorker.com/sitemap.xml","https://www.newyorker.com/tagpages-sitemap.xml","https://www.newyorker.com/categories-sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["Organization","WebSite","ItemList"],"feeds":1,"feed_urls":["https://www.newyorker.com/feed/rss"],"api_endpoints_seen":["www.newyorker.com/auth/refresh","paywall.sophi.io/hosts/www.newyorker.com/decisions/me","cdn.gladly.com/orgs/configs/chat/condenast.com-The_New_Yorker.json","cdn.gladly.com/orgs/configs/chat/condenast.com-The_New_Yorker-dynamic.json"],"inner_page":{"url":"https://www.newyorker.com/books/this-week-in-fiction/kevin-barry-10-19-26","status":200,"json_ld_types":["NewsArticle","BreadcrumbList"],"skipped":null,"sitemap":"https://www.newyorker.com/feed/google-news-sitemap-feed/sitemap-google-news"}},"performance":{"response_ms":143,"bytes":400000,"redirects":0,"cookies_set":5},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 9 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/newyorker.com.json"},{"schema":"ratradar.site/v1","host":"www.vox.com","name":"Vox","category":"news","country":"United States","verdict":"open","hostility_score":8,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.vox.com/sitemaps/google_news","Read the JSON-LD on the page (NewsMediaOrganization, WebSite). It is cleaner than the markup.","The site publishes 1 RSS or Atom feed. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example concertads-configs.vox-cdn.com/voxmedia/duet/vox/config.json. Untested, check the terms before using them."],"risks":["robots.txt forbids 11 of 14 known AI crawlers (ChatGPT-User, ClaudeBot, Claude-Web, anthropic-ai, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly","Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":15,"crawl_delay_seconds":null,"ai_crawlers_blocked":["ChatGPT-User","ClaudeBot","Claude-Web","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Bytespider","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":11,"sitemap_urls":["https://www.vox.com/sitemaps/google_news","https://www.vox.com/sitemaps"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["NewsMediaOrganization","WebSite"],"feeds":1,"feed_urls":["https://www.vox.com/rss/index.xml"],"api_endpoints_seen":["concertads-configs.vox-cdn.com/voxmedia/duet/vox/config.json","cdn.concert.io/lookup/aHR0cHM6Ly93d3cudm94LmNvbS8=.json","api.parsely.com/v2/related"],"inner_page":{"url":"https://www.vox.com/podcasts/505705/free-speech-first-amendment-trump-charlie-kirk-press-ban-fire","status":200,"json_ld_types":["NewsArticle","BreadcrumbList"],"skipped":null,"sitemap":"https://www.vox.com/sitemaps/google_news"}},"performance":{"response_ms":147,"bytes":400000,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"Fastly detected","points":2},{"label":"Cloudflare seen by the browser","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 11 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/vox.com.json"},{"schema":"ratradar.site/v1","host":"www.buzzfeed.com","name":"BuzzFeed","category":"news","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.buzzfeed.com/sitemap/buzzfeed.xml","A browser load called JSON endpoints that look like data, for example abeagle-public.buzzfeed.com/public/v3/experiment_variants. Untested, check the terms before using them."],"risks":["robots.txt forbids 12 of 14 known AI crawlers (GPTBot, ClaudeBot, Claude-Web, anthropic-ai, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":25,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ClaudeBot","Claude-Web","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Bytespider","Amazonbot","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":12,"sitemap_urls":["https://www.buzzfeed.com/sitemap/buzzfeed.xml","https://www.buzzfeed.com/sitemap/tasty.xml","https://www.buzzfeed.com/sitemap/video.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["abeagle-public.buzzfeed.com/public/v3/experiment_variants","lexicon.33across.com/v1/envelope","floors.atmtd.com/floors.json"],"inner_page":{"url":"https://www.buzzfeed.com/jp/shinobuuchihara/easy-aburaage-recipes","status":200,"json_ld_types":["NewsArticle","Organization","ItemList"],"skipped":null,"sitemap":"https://www.buzzfeed.com/sitemap/news/ja-jp.xml"}},"performance":{"response_ms":49,"bytes":400000,"redirects":0,"cookies_set":5},"why_this_score":[{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 12 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/buzzfeed.com.json"},{"schema":"ratradar.site/v1","host":"www.huffpost.com","name":"HuffPost","category":"news","country":"United States","verdict":"open","hostility_score":8,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.huffpost.com/static-assets/isolated/huffpostsitemapgeneratorjob-prod-public/us/sitemaps/sitemap-v1.xml","Read the JSON-LD on the page (NewsMediaOrganization, WebSite, WebPage). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example abeagle-public.buzzfeed.com/v3/experiment_variants. Untested, check the terms before using them."],"risks":["robots.txt forbids 14 of 14 known AI crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly","Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":9,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","Claude-Web","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Bytespider","Amazonbot","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":14,"sitemap_urls":["https://www.huffpost.com/static-assets/isolated/huffpostsitemapgeneratorjob-prod-public/us/sitemaps/sitemap-v1.xml","https://www.huffpost.com/static-assets/isolated/huffpostsitemapgeneratorjob-prod-public/us/sitemaps/sitemap-google-news.xml","https://www.huffpost.com/static-assets/isolated/huffpostsitemapgeneratorjob-prod-public/us/sitemaps/sitemap-google-video.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["NewsMediaOrganization","WebSite","WebPage"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["abeagle-public.buzzfeed.com/v3/experiment_variants","www.huffpost.com/client/campaigns"],"inner_page":{"url":"https://www.huffpost.com/news/topic/donald-trump","status":200,"json_ld_types":["NewsMediaOrganization","WebSite","CollectionPage"],"skipped":null,"sitemap":"https://www.huffpost.com/static-assets/isolated/huffpostsitemapgeneratorjob-prod-public/us/sitemaps/sections.xml"}},"performance":{"response_ms":109,"bytes":400000,"redirects":0,"cookies_set":3},"why_this_score":[{"label":"Fastly detected","points":2},{"label":"Cloudflare seen by the browser","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 14 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/huffpost.com.json"},{"schema":"ratradar.site/v1","host":"www.nbcnews.com","name":"NBC News","category":"news","country":"United States","verdict":"guarded","hostility_score":36,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429.","Enumerate pages from the sitemap: https://www.nbcnews.com/sitemap/nbcnews/sitemap-index","Read the JSON-LD on the page (CollectionPage, Dataset, WebSite). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example c2.piano.io/xbuilder/experience/execute. Untested, check the terms before using them."],"risks":["robots.txt forbids 14 of 14 known AI crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai","DataDome","Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":12,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","Claude-Web","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Bytespider","Amazonbot","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":14,"sitemap_urls":["https://www.nbcnews.com/sitemap/nbcnews/sitemap-index","https://www.nbcnews.com/sitemap/nbcnews/sitemap-news","https://www.nbcnews.com/sitemap/nbcnews/sitemap-curations"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["CollectionPage","Dataset","WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["c2.piano.io/xbuilder/experience/execute","www.nbcnews.com/bentoapi/broadcastSchedules","identity.mparticle.com/v1/identify"],"inner_page":{"url":"https://www.nbcnews.com/news/us-news/philadelphia-police-officer-killed-another-wounded-responding-reports-rcna602711","status":200,"json_ld_types":["NewsArticle","Dataset"],"skipped":null,"sitemap":"https://www.nbcnews.com/sitemap/nbcnews/sitemap-news"}},"performance":{"response_ms":68,"bytes":400000,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"Akamai detected","points":10},{"label":"DataDome seen by the browser","points":20},{"label":"Cloudflare seen by the browser","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 14 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/nbcnews.com.json"},{"schema":"ratradar.site/v1","host":"www.cbsnews.com","name":"CBS News","category":"news","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.cbsnews.com/xml-sitemap/index.xml","Read the JSON-LD on the page (NewsMediaOrganization, WebSite, BreadcrumbList, CollectionPage). It is cleaner than the markup.","The site publishes 1 RSS or Atom feed. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example www.cbsnews.com/feedfiles/breakingnews_us.rundown.json. Untested, check the terms before using them."],"risks":["robots.txt forbids some AI crawlers: GPTBot."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":35,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot"],"ai_crawlers_blocked_count":1,"sitemap_urls":["https://www.cbsnews.com/xml-sitemap/index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["NewsMediaOrganization","WebSite","BreadcrumbList","CollectionPage"],"feeds":1,"feed_urls":["https://www.cbsnews.com/latest/rss/main/"],"api_endpoints_seen":["www.cbsnews.com/feedfiles/breakingnews_us.rundown.json","cdn.privacy.paramount.com/config/menu-web.json","www.cbsnews.com/live/channel-data.json"],"inner_page":{"url":"https://www.cbsnews.com/texas/news/dallas-child-accidental-deadly-shooting-10-11-2026/","status":200,"json_ld_types":["NewsMediaOrganization","NewsArticle","WebPage","BreadcrumbList"],"skipped":null,"sitemap":"https://www.cbsnews.com/xml-sitemap/news.xml"}},"performance":{"response_ms":46,"bytes":400000,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 1 AI crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/cbsnews.com.json"},{"schema":"ratradar.site/v1","host":"abcnews.go.com","name":"ABC News","category":"news","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://abcnews.go.com/xmap","Read the JSON-LD on the page (WebSite, WebPage). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example pcs.bamgrid.com/v1/abc_news. Untested, check the terms before using them."],"risks":["robots.txt forbids 6 of 14 known AI crawlers (GPTBot, ChatGPT-User, anthropic-ai, CCBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"nginx/1.28.3"},"robots":{"found":true,"disallow_all":false,"disallow_rules":44,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","anthropic-ai","CCBot","Google-Extended","Bytespider"],"ai_crawlers_blocked_count":6,"sitemap_urls":["https://abcnews.go.com/xmap","https://abcnews.go.com/xmlLatestStories","https://abcnews.go.com/xmlLatestVideos"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebSite","WebPage"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["pcs.bamgrid.com/v1/abc_news","abcnews.com/proxy/distro/getConfig"],"inner_page":null},"performance":{"response_ms":298,"bytes":400000,"redirects":1,"cookies_set":4},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 6 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/abcnews.go.com.json"},{"schema":"ratradar.site/v1","host":"www.latimes.com","name":"Los Angeles Times","category":"news","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Enumerate pages from the sitemap: https://www.latimes.com/sitemaps/sitemap.xml","Read the JSON-LD on the page (WebPage). It is cleaner than the markup.","The site publishes 1 RSS or Atom feed. Prefer them for new items."],"risks":["Hostile to automation. Keep volumes low and cache.","robots.txt forbids 3 of 14 known AI crawlers (GPTBot, ClaudeBot, Bytespider). An agent that identifies as an AI crawler should not fetch this site.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"N/A"},"robots":{"found":true,"disallow_all":false,"disallow_rules":20,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ClaudeBot","Bytespider"],"ai_crawlers_blocked_count":3,"sitemap_urls":["https://www.latimes.com/sitemaps/sitemap.xml","https://www.latimes.com/news-sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebPage"],"feeds":1,"feed_urls":["https://www.latimes.com/index.rss"],"api_endpoints_seen":[],"inner_page":{"url":"https://www.latimes.com/sports/highschool/story/2026-10-11/volunteers-fill-multiple-roles-to-help-high-school-teams-succeed","status":200,"json_ld_types":["NewsArticle"],"skipped":null,"sitemap":"https://www.latimes.com/news-sitemap-latest.xml"}},"performance":{"response_ms":56,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"CloudFront detected","points":2},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 3 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/latimes.com.json"},{"schema":"ratradar.site/v1","host":"www.chicagotribune.com","name":"Chicago Tribune","category":"news","country":"United States","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.chicagotribune.com/sitemap.xml","Read the JSON-LD on the page (NewsMediaOrganization, LocalBusiness). It is cleaner than the markup.","The site publishes 2 RSS or Atom feeds. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example session.chicagotribune.com/api/session. Untested, check the terms before using them."],"risks":["robots.txt forbids 13 of 14 known AI crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":8,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","Claude-Web","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Bytespider","Amazonbot","Applebot-Extended","Meta-ExternalAgent"],"ai_crawlers_blocked_count":13,"sitemap_urls":["https://www.chicagotribune.com/sitemap.xml","https://www.chicagotribune.com/image-sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["NewsMediaOrganization","LocalBusiness"],"feeds":2,"feed_urls":["https://www.chicagotribune.com/feed/","https://www.chicagotribune.com/comments/feed/"],"api_endpoints_seen":["session.chicagotribune.com/api/session","api.parsely.com/v2/profile","cmp.osano.com/16A1AnRt2Fn8i1unj/f15ebf08-7008-40fe-9af3-db96dc3e8266/en.json","r610.chicagotribune.com/DG/DEFAULT/rest/rpc/679","r610.chicagotribune.com/DG/DEFAULT/rest/rpc/679/LB-Zone-1","geo.privacymanager.io/"],"inner_page":{"url":"https://www.chicagotribune.com/2026/10/11/asking-eric-in-laws-get-into-altercation-on-vacation/","status":200,"json_ld_types":["NewsArticle","BreadcrumbList"],"skipped":null,"sitemap":"https://www.chicagotribune.com/sitemap.xml?yyyy=2026&mm=10&dd=11"}},"performance":{"response_ms":119,"bytes":276567,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Cloudflare seen by the browser","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 13 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/chicagotribune.com.json"},{"schema":"ratradar.site/v1","host":"seekingalpha.com","name":"Seeking Alpha","category":"finance","country":"United States","verdict":"open","hostility_score":22,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://seekingalpha.com/sitemap_news.xml","Read the JSON-LD on the page (Organization, WebSite, SoftwareApplication). It is cleaner than the markup."],"risks":["robots.txt forbids 6 of 14 known AI crawlers (ChatGPT-User, ClaudeBot, CCBot, Bytespider, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly","PerimeterX"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":21,"crawl_delay_seconds":null,"ai_crawlers_blocked":["ChatGPT-User","ClaudeBot","CCBot","Bytespider","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":6,"sitemap_urls":["https://seekingalpha.com/sitemap_news.xml","https://seekingalpha.com/instablog/index.xml","https://seekingalpha.com/news/index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["Organization","WebSite","SoftwareApplication"],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://seekingalpha.com/news/4649212-energy-sector-overshadows-in-q3-as","status":200,"json_ld_types":["NewsArticle"],"skipped":null,"sitemap":"https://seekingalpha.com/news/2026_9.xml"}},"performance":{"response_ms":489,"bytes":400000,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"Fastly detected","points":2},{"label":"PerimeterX seen by the browser","points":20}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 6 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/seekingalpha.com.json"},{"schema":"ratradar.site/v1","host":"www.fool.com","name":"Motley Fool","category":"finance","country":"United States","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.fool.com/sitemap/","Read the JSON-LD on the page (Organization, WebSite, WebPage, FAQPage). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example cdn-4.convertexperiments.com/api/v1/project-optional-settings/10045690/10046383. Untested, check the terms before using them."],"risks":["robots.txt forbids some AI crawlers: Bytespider."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":83,"crawl_delay_seconds":null,"ai_crawlers_blocked":["Bytespider"],"ai_crawlers_blocked_count":1,"sitemap_urls":["https://www.fool.com/sitemap/","https://www.fool.com/wag-sitemap.xml","https://www.fool.com/quotes-sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["Organization","WebSite","WebPage","FAQPage"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["cdn-4.convertexperiments.com/api/v1/project-optional-settings/10045690/10046383","jnn-pa.googleapis.com/$rpc/google.internal.waa.v1.Waa/GenerateIT"],"inner_page":{"url":"https://www.fool.com/investing/2026/10/11/id-put-2500-into-each-of-these-4-dividend-stocks/","status":200,"json_ld_types":["NewsArticle"],"skipped":null,"sitemap":"https://www.fool.com/news-sitemap.xml"}},"performance":{"response_ms":59,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 1 AI crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/fool.com.json"},{"schema":"ratradar.site/v1","host":"www.barrons.com","name":"Barrons","category":"finance","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"respect_robots_skip","steps":["robots.txt disallows every crawler. Do not scrape it. Look for an official API, a data license, or a partner feed.","Enumerate pages from the sitemap: https://www.barrons.com/bol_news_sitemap.xml"],"risks":["Scraping against an explicit robots.txt ban is the quickest way to a legal or blocking problem."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":false,"official_api_advised":true},"protections":{"vendors":["DataDome","CloudFront"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"CloudFront"},"robots":{"found":true,"disallow_all":true,"disallow_rules":1,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.barrons.com/bol_news_sitemap.xml","https://www.barrons.com/bol_live_news_sitemap.xml","https://www.barrons.com/barrons_video_recent.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":436,"bytes":771,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 401 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"DataDome detected","points":20},{"label":"CloudFront detected","points":2},{"label":"robots.txt disallows everything","points":10},{"label":"Raised to at least 50 because robots.txt disallows everything","points":0}],"eu_ai_training":{"level":"strong","signals":[{"kind":"robots_txt","label":"robots.txt bans every crawler"},{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":401,"headless_browser_status":401,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/barrons.com.json"},{"schema":"ratradar.site/v1","host":"www.cnbc.com","name":"CNBC","category":"finance","country":"United States","verdict":"open","hostility_score":16,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.cnbc.com/sitemapAll.xml"],"risks":["robots.txt forbids 14 of 14 known AI crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai","Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":8,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","Claude-Web","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Bytespider","Amazonbot","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":14,"sitemap_urls":["https://www.cnbc.com/sitemapAll.xml","https://www.cnbc.com/sitemap_news.xml","https://www.cnbc.com/sitemapvideoAll.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.cnbc.com/2026/10/11/ai-software-developer-jobs.html","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.cnbc.com/sitemap_news.xml"}},"performance":{"response_ms":270,"bytes":400000,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"Akamai detected","points":10},{"label":"Cloudflare seen by the browser","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 14 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/cnbc.com.json"},{"schema":"ratradar.site/v1","host":"www.morningstar.com","name":"Morningstar","category":"finance","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Enumerate pages from the sitemap: https://www.morningstar.com/sitemap.xml"],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["AWS WAF","CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"CloudFront"},"robots":{"found":true,"disallow_all":false,"disallow_rules":12,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.morningstar.com/sitemap.xml","https://www.morningstar.com/content/marketing_sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.morningstar.com/etfs/arcx/idu/quote","status":200,"json_ld_types":["Corporation","PriceSpecification"],"skipped":null,"sitemap":"https://www.morningstar.com/sitemaps/etfs_1.xml"}},"performance":{"response_ms":687,"bytes":400000,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"AWS WAF detected","points":12},{"label":"CloudFront detected","points":2},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/morningstar.com.json"},{"schema":"ratradar.site/v1","host":"www.tradingview.com","name":"TradingView","category":"finance","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.tradingview.com/sitemap.xml","Read the JSON-LD on the page (Organization). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.tradingview.com/check_language/. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"tv"},"robots":{"found":true,"disallow_all":false,"disallow_rules":15,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.tradingview.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.tradingview.com/check_language/","news-mediator.tradingview.com/public/news-flow/v2/news","scanner.tradingview.com/america/scan","scanner.tradingview.com/symbol"],"inner_page":{"url":"https://www.tradingview.com/news/tradingview:44303bf92e18e:0-ascelia-pharma-ab-publ-ace-says-fda-may-accept-sparkle-reanalysis-no-new-study-planned/","status":200,"json_ld_types":["NewsArticle","BreadcrumbList"],"skipped":null,"sitemap":"https://www.tradingview.com/sitemap-news.xml"}},"performance":{"response_ms":51,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/tradingview.com.json"},{"schema":"ratradar.site/v1","host":"www.kraken.com","name":"Kraken","category":"finance","country":"United States","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.kraken.com/sitemaps/sitemap.xml","Read the JSON-LD on the page (Organization, FinancialService, Service, WebSite). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example api.ipify.org/. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":2,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.kraken.com/sitemaps/sitemap.xml","https://www.kraken.com/sitemaps/sell-asset.xml","https://www.kraken.com/sitemaps/buy-asset.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["Organization","FinancialService","Service","WebSite","BreadcrumbList","WebPage"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["api.ipify.org/","sg.kraken.com/v1/initialize","iapi.kraken.com/api/internal/session/expiration"],"inner_page":{"url":"https://www.kraken.com/es/learn/videos/crypto-101","status":200,"json_ld_types":["Organization","FinancialService","Service","WebSite","BreadcrumbList","WebPage"],"skipped":null,"sitemap":"https://www.kraken.com/sitemaps/sitemap.xml"}},"performance":{"response_ms":77,"bytes":400000,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/kraken.com.json"},{"schema":"ratradar.site/v1","host":"etherscan.io","name":"Etherscan","category":"finance","country":"Singapore","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Enumerate pages from the sitemap: https://etherscan.io/sitemap"],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":8,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://etherscan.io/sitemap"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://etherscan.io/tx/0xf8074f3a46e85113d63038f64bf9e20d34b658ab5aaacff080f82cb166c6cbeb","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://etherscan.io/sitemaps/tx-1"}},"performance":{"response_ms":193,"bytes":141930,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/etherscan.io.json"},{"schema":"ratradar.site/v1","host":"www.coindesk.com","name":"CoinDesk","category":"finance","country":"United States","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.coindesk.com/sitemap-index.xml","A browser load called JSON endpoints that look like data, for example www.coindesk.com/api/cc-data-proxy/news/v1/search. Untested, check the terms before using them."],"risks":["robots.txt forbids 12 of 14 known AI crawlers (GPTBot, ClaudeBot, Claude-Web, anthropic-ai, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Vercel","server":"Vercel"},"robots":{"found":true,"disallow_all":false,"disallow_rules":42,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ClaudeBot","Claude-Web","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Bytespider","Amazonbot","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":12,"sitemap_urls":["https://www.coindesk.com/sitemap-index.xml","https://www.coindesk.com/es/sitemap-index.xml","https://www.coindesk.com/uk/sitemap-index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.coindesk.com/api/cc-data-proxy/news/v1/search","www.coindesk.com/api/cc-data-proxy/asset/v1/search","www.coindesk.com/api/v1/sig"],"inner_page":{"url":"https://www.coindesk.com/business/2026/10/11/imf-finds-demand-for-tokenized-stocks-says-the-market-is-still-volatile-illiquid","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.coindesk.com/arc/outboundfeeds/news-sitemap-index"}},"performance":{"response_ms":710,"bytes":400000,"redirects":0,"cookies_set":14},"why_this_score":[{"label":"Cloudflare seen by the browser","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 12 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/coindesk.com.json"},{"schema":"ratradar.site/v1","host":"www.nerdwallet.com","name":"NerdWallet","category":"finance","country":"United States","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.nerdwallet.com/sitemaps/us/wp-sitemap.xml","Read the JSON-LD on the page (Organization, WebSite). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example siteusagetools.nerdwallet.com/v1/projects/EIeR29eZ4CoiwV5W9aXGFqrKP0BtWoJT/settings. Untested, check the terms before using them."],"risks":["robots.txt forbids some AI crawlers: Google-Extended."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":37,"crawl_delay_seconds":null,"ai_crawlers_blocked":["Google-Extended"],"ai_crawlers_blocked_count":1,"sitemap_urls":["https://www.nerdwallet.com/sitemaps/us/wp-sitemap.xml","https://www.nerdwallet.com/sitemaps/us/wp-sitemap-news-articles-1.xml","https://www.nerdwallet.com/sitemaps/ca/wp-sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["Organization","WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["siteusagetools.nerdwallet.com/v1/projects/EIeR29eZ4CoiwV5W9aXGFqrKP0BtWoJT/settings","browser-intake-datadoghq.com/api/v2/rum","siteusage.nerdwallet.com/v1/p","siteusage.nerdwallet.com/v1/t"],"inner_page":null},"performance":{"response_ms":176,"bytes":400000,"redirects":0,"cookies_set":7},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 1 AI crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/nerdwallet.com.json"},{"schema":"ratradar.site/v1","host":"www.bankrate.com","name":"Bankrate","category":"finance","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.bankrate.com/sitemap/sitemap-index.xml","Read the JSON-LD on the page (WebPage). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.bankrate.com/api/member/profile/. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":42,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.bankrate.com/sitemap/sitemap-index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["WebPage"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.bankrate.com/api/member/profile/","www.bankrate.com/monarch/api/v2/sources/18b02819-56c9-4af5-8780-faf726c1ba86/evaluations","www.bankrate.com/monarch/api/v2/sources/18b02819-56c9-4af5-8780-faf726c1ba86/rule-sets/feedbackComponentDisplay/evaluations","www.bankrate.com/monarch/api/v2/sources/18b02819-56c9-4af5-8780-faf726c1ba86/rule-sets/rippleBannersAndNudgesDisplay/evaluations"],"inner_page":{"url":"https://www.bankrate.com/credit-cards/news/balance-transfer-offers-heating-up/","status":200,"json_ld_types":["WebPage","Article","BreadcrumbList"],"skipped":null,"sitemap":"https://www.bankrate.com/sitemap/wp-post-1.xml"}},"performance":{"response_ms":52,"bytes":400000,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/bankrate.com.json"},{"schema":"ratradar.site/v1","host":"www.hubspot.com","name":"HubSpot","category":"saas","country":"United States","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.hubspot.com/sitemap.xml","Read the JSON-LD on the page (Organization, WebSite, WebPage, Product). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example wtcfns.hubspot.com/wt-api/current-user. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":356,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["Organization","WebSite","WebPage","Product"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["wtcfns.hubspot.com/wt-api/current-user","cta-service-cms2.hubspot.com/web-interactives/public/v1/embed/combinedConfigs","www.hubspot.com/_hcms/livechat/widget","c.amazon-adsystem.com/aat/allowlist/allowlist.json"],"inner_page":{"url":"https://www.hubspot.com/products/artificial-intelligence/use-cases/capture-and-qualify-sales-leads","status":200,"json_ld_types":["BreadcrumbList"],"skipped":null,"sitemap":"https://www.hubspot.com/sitemap.xml"}},"performance":{"response_ms":1490,"bytes":400000,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/hubspot.com.json"},{"schema":"ratradar.site/v1","host":"www.salesforce.com","name":"Salesforce","category":"saas","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Enumerate pages from the sitemap: https://www.salesforce.com/sitemap.xml","Read the JSON-LD on the page (Organization). It is cleaner than the markup."],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":181,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.salesforce.com/sitemap.xml","https://www.salesforce.com/ap/sitemap.xml","https://www.salesforce.com/au/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.salesforce.com/news/092021-df21-homepage-block-ql/","status":200,"json_ld_types":["WebPage","WebSite","Organization"],"skipped":null,"sitemap":"https://www.salesforce.com/news/page-sitemap.xml"}},"performance":{"response_ms":160,"bytes":400000,"redirects":1,"cookies_set":7},"why_this_score":[{"label":"Akamai detected","points":10},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/salesforce.com.json"},{"schema":"ratradar.site/v1","host":"www.intercom.com","name":"Intercom","category":"saas","country":"Ireland","verdict":"open","hostility_score":24,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.intercom.com/sitemap.xml","Read the JSON-LD on the page (WebSite, Organization, WebPage, Product). It is cleaner than the markup."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai","AWS WAF","CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"Vercel"},"robots":{"found":true,"disallow_all":false,"disallow_rules":7,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.intercom.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["WebSite","Organization","WebPage","Product","SoftwareApplication","ItemList","BreadcrumbList"],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.intercom.com/blog/videos/built-for-you-202404-fin-ai-copilot","status":200,"json_ld_types":["BlogPosting","WebPage","ImageObject","BreadcrumbList","WebSite","Organization","Person"],"skipped":null,"sitemap":"https://www.intercom.com/sitemap-blog.xml"}},"performance":{"response_ms":425,"bytes":400000,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"Akamai detected","points":10},{"label":"AWS WAF detected","points":12},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/intercom.com.json"},{"schema":"ratradar.site/v1","host":"squareup.com","name":"Square","category":"finance","country":"United States","verdict":"guarded","hostility_score":26,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429.","Enumerate pages from the sitemap: https://squareup.com/us/en/sitemap.xml","A browser load called JSON endpoints that look like data, for example squareup.com/mp/status. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare","Kasada"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":39,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://squareup.com/us/en/sitemap.xml","https://squareup.com/us/es/sitemap.xml","https://squareup.com/ca/en/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["squareup.com/mp/status","squareup.com/v1/cdp/batch","squareup.com/shop/hardware/us/en/api/v1/item_count"],"inner_page":{"url":"https://squareup.com/us/en/events/squarecornerstore","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://squareup.com/us/en/sitemap.xml"}},"performance":{"response_ms":279,"bytes":400000,"redirects":1,"cookies_set":14},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"Kasada detected","points":20}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/squareup.com.json"},{"schema":"ratradar.site/v1","host":"plaid.com","name":"Plaid","category":"finance","country":"United States","verdict":"open","hostility_score":8,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://plaid.com/sitemap.xml","Read the JSON-LD on the page (organization). It is cleaner than the markup."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront","Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"AmazonS3"},"robots":{"found":true,"disallow_all":false,"disallow_rules":16,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://plaid.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://plaid.com/blog/tag/company/","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://plaid.com/sitemap.xml"}},"performance":{"response_ms":168,"bytes":249709,"redirects":0,"cookies_set":4},"why_this_score":[{"label":"CloudFront detected","points":2},{"label":"Cloudflare seen by the browser","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/plaid.com.json"},{"schema":"ratradar.site/v1","host":"www.revolut.com","name":"Revolut","category":"finance","country":"United Kingdom","verdict":"blocked","hostility_score":76,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":57,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.revolut.com/sitemap-index.xml"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":79,"bytes":400000,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/revolut.com.json"},{"schema":"ratradar.site/v1","host":"monzo.com","name":"Monzo","category":"finance","country":"United Kingdom","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://monzo.com/sitemap-index.xml","Read the JSON-LD on the page (Organization). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example monzo.com/_next/data/0JBtVELZWPQRQPXxW7PjR/gb/sign-up.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":10,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://monzo.com/sitemap-index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["monzo.com/_next/data/0JBtVELZWPQRQPXxW7PjR/gb/sign-up.json","monzo.com/_next/data/0JBtVELZWPQRQPXxW7PjR/gb/business-banking.json","monzo.com/_next/data/0JBtVELZWPQRQPXxW7PjR/gb.json","api.monzo.com/oauth2/token"],"inner_page":{"url":"https://monzo.com/blog/2015/06/01/welcome-to-mondo","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://monzo.com/gb-sitemap.xml"}},"performance":{"response_ms":94,"bytes":400000,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/monzo.com.json"},{"schema":"ratradar.site/v1","host":"www.klarna.com","name":"Klarna","category":"finance","country":"Sweden","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Enumerate pages from the sitemap: https://www.klarna.com/sitemaps/sitemap-index.xml"],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"envoy"},"robots":{"found":true,"disallow_all":false,"disallow_rules":40,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.klarna.com/sitemaps/sitemap-index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.klarna.com/us/articles/bnpl-vs-klarna-credit-card/","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.klarna.com/us/api/klarna-web-bff/api/sitemap/us/sitemap.xml"}},"performance":{"response_ms":488,"bytes":400000,"redirects":1,"cookies_set":3},"why_this_score":[{"label":"CloudFront detected","points":2},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/klarna.com.json"},{"schema":"ratradar.site/v1","host":"www.adyen.com","name":"Adyen","category":"finance","country":"Netherlands","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.adyen.com/sitemap_index.xml","Read the JSON-LD on the page (Organization). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.adyen.com/_i18n/234c4e76/en/messages.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Netlify","server":"Netlify"},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.adyen.com/sitemap_index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.adyen.com/_i18n/234c4e76/en/messages.json"],"inner_page":{"url":"https://www.adyen.com/events/inside-adyen-tokyo","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.adyen.com/sitemap-en.xml"}},"performance":{"response_ms":280,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/adyen.com.json"},{"schema":"ratradar.site/v1","host":"www.priceline.com","name":"Priceline","category":"travel","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Enumerate pages from the sitemap: https://www.priceline.com/sitemap.xml","Read the JSON-LD on the page (WebSite, Organization, TravelAgency). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.priceline.com/9aTjSd0n/xhr/assets/js/bundle. Untested, check the terms before using them."],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare","Fastly","PerimeterX"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":121,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebSite","Organization","TravelAgency"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.priceline.com/9aTjSd0n/xhr/assets/js/bundle"],"inner_page":{"url":"https://www.priceline.com/hotel-deals/en-us/P3000015284/hotels-in-las-vegas.ssp","status":200,"json_ld_types":["TravelAgency"],"skipped":null,"sitemap":"https://www.priceline.com/sitemaps/hotels.xml.gz"}},"performance":{"response_ms":586,"bytes":400000,"redirects":0,"cookies_set":6},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"Fastly detected","points":2},{"label":"PerimeterX seen by the browser","points":20},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/priceline.com.json"},{"schema":"ratradar.site/v1","host":"www.hopper.com","name":"Hopper","category":"travel","country":"Canada","verdict":"open","hostility_score":15,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls.","Enumerate pages from the sitemap: https://www.hopper.com/sitemap.xml"],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"istio-envoy"},"robots":{"found":true,"disallow_all":false,"disallow_rules":32,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.hopper.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://hopper.com/fr/product/price-prediction","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.hopper.com/sitemap.xml"}},"performance":{"response_ms":251,"bytes":54165,"redirects":0,"cookies_set":5},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/hopper.com.json"},{"schema":"ratradar.site/v1","host":"www.momondo.com","name":"Momondo","category":"travel","country":"Denmark","verdict":"open","hostility_score":4,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.momondo.com/sitemap.xml","Read the JSON-LD on the page (Website, Organization, FAQPage). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.momondo.com/s/run/fpc/context. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly","CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":"KAYAK/1.0"},"robots":{"found":true,"disallow_all":false,"disallow_rules":269,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.momondo.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["Website","Organization","FAQPage"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.momondo.com/s/run/fpc/context","www.momondo.com/mvm/p13n/flight/destinations","www.momondo.com/s/horizon/react/component/CurrencyPickerStateProviderAction","www.momondo.com/a/api/smarty/nearby","www.momondo.com/s/horizon/common/privacy/providers/CookiesDataProvider","www.momondo.com/i/api/trips/user/v1/currentUser"],"inner_page":{"url":"https://www.momondo.com/hotels","status":200,"json_ld_types":["FAQPage","Website","Organization"],"skipped":null,"sitemap":"https://www.momondo.com/sitemaps/momondo-kyk-wl-us-hotel_frontdoor_unknown.xml.gz"}},"performance":{"response_ms":908,"bytes":400000,"redirects":0,"cookies_set":10},"why_this_score":[{"label":"Fastly detected","points":2},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/momondo.com.json"},{"schema":"ratradar.site/v1","host":"www.kiwi.com","name":"Kiwi.com","category":"travel","country":"Czechia","verdict":"open","hostility_score":10,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.kiwi.com/sitemap.xml","Read the JSON-LD on the page (Organization). It is cleaner than the markup."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare","Fastly","CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":1309,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.kiwi.com/sitemap.xml","https://www.kiwi.com/stories/sitemap.xml","https://www.kiwi.com/help/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.kiwi.com/en/region/poi/glacier-national-park/","status":200,"json_ld_types":["FAQPage","Organization","WebSite","WebPage","TouristAttraction","ImageObject"],"skipped":null,"sitemap":"https://www.kiwi.com/odcf/poi/sitemap/0.xml"}},"performance":{"response_ms":971,"bytes":400000,"redirects":1,"cookies_set":5},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"Fastly detected","points":2},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":0,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/kiwi.com.json"},{"schema":"ratradar.site/v1","host":"www.trivago.com","name":"Trivago","category":"travel","country":"Germany","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies.","Enumerate pages from the sitemap: https://www.trivago.com/custom-xml-sitemaps/en-US/sitemap-accommodations-index.xml","A browser load called JSON endpoints that look like data, for example akamai-errorcodes.dus.tcs.trivago.cloud/prod_www/error/0.a22c2d17.1791729482.cfe151d. Untested, check the terms before using them."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans.","robots.txt forbids some AI crawlers: ClaudeBot."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Akamai"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"AkamaiGHost"},"robots":{"found":true,"disallow_all":false,"disallow_rules":104,"crawl_delay_seconds":null,"ai_crawlers_blocked":["ClaudeBot"],"ai_crawlers_blocked_count":1,"sitemap_urls":["https://www.trivago.com/custom-xml-sitemaps/en-US/sitemap-accommodations-index.xml","https://www.trivago.com/custom-xml-sitemaps/en-US/sitemap-destinations-index.xml","https://www.trivago.com/custom-xml-sitemaps/en-US/sitemap-points-of-interest-index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["akamai-errorcodes.dus.tcs.trivago.cloud/prod_www/error/0.a22c2d17.1791729482.cfe151d"],"inner_page":null},"performance":{"response_ms":47,"bytes":3828,"redirects":0,"cookies_set":4},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"strong","signals":[{"kind":"robots_txt","label":"robots.txt blocks 1 AI crawler"},{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/trivago.com.json"},{"schema":"ratradar.site/v1","host":"www.hotwire.com","name":"Hotwire","category":"travel","country":"United States","verdict":"guarded","hostility_score":25,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls.","Enumerate pages from the sitemap: https://www.hotwire.com/sitemap.xml"],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":35,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.hotwire.com/hotels/","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.hotwire.com/sitemap.xml"}},"performance":{"response_ms":1120,"bytes":6804,"redirects":0,"cookies_set":6},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15},{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/hotwire.com.json"},{"schema":"ratradar.site/v1","host":"turo.com","name":"Turo","category":"travel","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Read the JSON-LD on the page (Organization, WebSite). It is cleaner than the markup."],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":57,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://turo.com/sitemap-index.xml.gz","https://help.turo.com/sitemap.xml"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":["Organization","WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":173,"bytes":400000,"redirects":0,"cookies_set":4},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/turo.com.json"},{"schema":"ratradar.site/v1","host":"www.rentalcars.com","name":"Rentalcars","category":"travel","country":"United Kingdom","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Enumerate pages from the sitemap: https://www.rentalcars.com/https_sitemapindex.xml","Read the JSON-LD on the page (FAQPage). It is cleaner than the markup."],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["AWS WAF","CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"istio-envoy"},"robots":{"found":true,"disallow_all":false,"disallow_rules":105,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.rentalcars.com/https_sitemapindex.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["FAQPage"],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.rentalcars.com/ar/airport/us/atl/","status":200,"json_ld_types":["BreadcrumbList"],"skipped":null,"sitemap":"https://www.rentalcars.com/https_sitemap_ar.xml"}},"performance":{"response_ms":496,"bytes":367750,"redirects":0,"cookies_set":6},"why_this_score":[{"label":"AWS WAF detected","points":12},{"label":"CloudFront detected","points":2},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/rentalcars.com.json"},{"schema":"ratradar.site/v1","host":"www.hertz.com","name":"Hertz","category":"travel","country":"United States","verdict":"guarded","hostility_score":35,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls.","Enumerate pages from the sitemap: https://www.hertz.com/hertz-sitemap.xml","Read the JSON-LD on the page (BreadcrumbList). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.hertz.com/A-thou-doe-prospeeceiud-accome-Hauen-heeleepell-. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Imperva","Fastly","Cloudflare"],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":14,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.hertz.com/hertz-sitemap.xml","https://www.hertz.com/us/en.sitemap-index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["BreadcrumbList"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.hertz.com/A-thou-doe-prospeeceiud-accome-Hauen-heeleepell-"],"inner_page":{"url":"https://www.hertz.com/p/collections/dream","status":200,"json_ld_types":["BreadcrumbList"],"skipped":null,"sitemap":"https://www.hertz.com/hertz-sitemap.xml"}},"performance":{"response_ms":376,"bytes":14054,"redirects":2,"cookies_set":6},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15},{"label":"Imperva detected","points":12},{"label":"Fastly detected","points":2},{"label":"Cloudflare seen by the browser","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/hertz.com.json"},{"schema":"ratradar.site/v1","host":"www.avis.com","name":"Avis","category":"travel","country":"United States","verdict":"guarded","hostility_score":28,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429.","Enumerate pages from the sitemap: https://www.avis.com/sitemap_index.xml","Read the JSON-LD on the page (WebPage, Organization, MemberProgram). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.avis.com/web/customer/session. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["DataDome","CloudFront","Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"envoy"},"robots":{"found":true,"disallow_all":false,"disallow_rules":9,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.avis.com/sitemap_index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["WebPage","Organization","MemberProgram"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.avis.com/web/customer/session","www.avis.com/js/","edge.adobedc.net/ee/or2/v1/interact"],"inner_page":{"url":"https://www.avis.com/en/products-and-services/products/childsafetyseats","status":200,"json_ld_types":["BreadcrumbList"],"skipped":null,"sitemap":"https://www.avis.com/en/sitemap.xml"}},"performance":{"response_ms":311,"bytes":400000,"redirects":1,"cookies_set":1},"why_this_score":[{"label":"DataDome detected","points":20},{"label":"CloudFront detected","points":2},{"label":"Cloudflare seen by the browser","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/avis.com.json"},{"schema":"ratradar.site/v1","host":"www.jetblue.com","name":"JetBlue","category":"travel","country":"United States","verdict":"open","hostility_score":8,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","A browser load called JSON endpoints that look like data, for example azrest.jetblue.com/od/od-service/nearby-airports. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly","Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":""},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["azrest.jetblue.com/od/od-service/nearby-airports","www.jetblue.com/api/gatekeeper","www.jetblue.com/c23f7ab876ac167e0546/projects/jb-auth/auth-wrapper/production.json","accounts.jetblue.com/api/v1/sessions/me"],"inner_page":null},"performance":{"response_ms":298,"bytes":400000,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"Fastly detected","points":2},{"label":"Cloudflare seen by the browser","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/jetblue.com.json"},{"schema":"ratradar.site/v1","host":"www.alaskaair.com","name":"Alaska Airlines","category":"travel","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Enumerate pages from the sitemap: https://www.alaskaair.com/sitemap.xml"],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":58,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.alaskaair.com/content/about-us/esg/people-communities/communities/aviation-day","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.alaskaair.com/sitemap.xml"}},"performance":{"response_ms":739,"bytes":400000,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"Fastly detected","points":2},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/alaskaair.com.json"},{"schema":"ratradar.site/v1","host":"wwws.airfrance.us","name":"Air France","category":"travel","country":"France","verdict":"guarded","hostility_score":45,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":15036,"bytes":null,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"No answer to a plain request","points":40},{"label":"Slow response (15036 ms)","points":5}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":0,"headless_browser_status":0,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/wwws.airfrance.us.json"},{"schema":"ratradar.site/v1","host":"www.britishairways.com","name":"British Airways","category":"travel","country":"United Kingdom","verdict":"guarded","hostility_score":45,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":15060,"bytes":null,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"No answer to a plain request","points":40},{"label":"Slow response (15060 ms)","points":5}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":0,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/britishairways.com.json"},{"schema":"ratradar.site/v1","host":"www.qatarairways.com","name":"Qatar Airways","category":"travel","country":"United Arab Emirates","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Akamai"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"AkamaiGHost"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":38,"bytes":374,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/qatarairways.com.json"},{"schema":"ratradar.site/v1","host":"www.singaporeair.com","name":"Singapore Airlines","category":"travel","country":"Singapore","verdict":"guarded","hostility_score":45,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"AkamaiGHost"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":87,"bytes":11912,"redirects":0,"cookies_set":5},"why_this_score":[{"label":"HTTP 404 on a plain request","points":35},{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":404,"headless_browser_status":404,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/singaporeair.com.json"},{"schema":"ratradar.site/v1","host":"www.greyhound.com","name":"Greyhound","category":"travel","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.greyhound.com/sitemap/sitemap-v2.xml","Read the JSON-LD on the page (WebSite). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example api.cms.flixbus.com/gql. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":5,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.greyhound.com/sitemap/sitemap-v2.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["api.cms.flixbus.com/gql","d3k6pebee3cv6.cloudfront.net/search-mask/current/asset-manifest.json","react-content-components.cms.flixbus.com/master/react-bundle/manifest-KcReactBundle.json","www.greyhound.com/api/nearby-city"],"inner_page":{"url":"https://www.greyhound.com/company/get-to-know-us","status":200,"json_ld_types":["BreadcrumbList"],"skipped":null,"sitemap":"https://www.greyhound.com/sitemap/sitemap-v2.xml/content/0"}},"performance":{"response_ms":50,"bytes":167494,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/greyhound.com.json"},{"schema":"ratradar.site/v1","host":"www.flixbus.com","name":"FlixBus","category":"travel","country":"Germany","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.flixbus.com/sitemap/sitemap-v2.xml","Read the JSON-LD on the page (WebSite). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example api.cms.flixbus.com/gql. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":5,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.flixbus.com/sitemap/sitemap-v2.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["api.cms.flixbus.com/gql","react-content-components.cms.flixbus.com/master/react-bundle/manifest-KcReactBundle.json","shop.flixbus.com/api/customer-auth/session/status","d3k6pebee3cv6.cloudfront.net/search-mask/current/asset-manifest.json"],"inner_page":{"url":"https://www.flixbus.com/company/prices/baggage-price-list","status":200,"json_ld_types":["BreadcrumbList"],"skipped":null,"sitemap":"https://www.flixbus.com/sitemap/sitemap-v2.xml/content/0"}},"performance":{"response_ms":48,"bytes":201329,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/flixbus.com.json"},{"schema":"ratradar.site/v1","host":"www.trainline.com","name":"Trainline","category":"travel","country":"United Kingdom","verdict":"open","hostility_score":22,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.thetrainline.com/live/sitemap","Read the JSON-LD on the page (WebSite, Organization). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example dd.thetrainline.com/js/. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["DataDome","CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"CloudFront"},"robots":{"found":true,"disallow_all":false,"disallow_rules":50,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.thetrainline.com/live/sitemap","https://www.thetrainline.com/sitemap.xml","https://www.thetrainline.com/sitemap-cms"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["WebSite","Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["dd.thetrainline.com/js/","www.thetrainline.com/api/locations-search/v1/search","www.thetrainline.com/api/basket","www.thetrainline.com/api/locations-search/v2/search","www.thetrainline.com/carrier-content/manifests/v2/web","www.thetrainline.com/version"],"inner_page":{"url":"https://www.thetrainline.com/live/departures/abbey-wood","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.thetrainline.com/live/sitemap"}},"performance":{"response_ms":556,"bytes":400000,"redirects":1,"cookies_set":4},"why_this_score":[{"label":"DataDome detected","points":20},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/trainline.com.json"},{"schema":"ratradar.site/v1","host":"www.homes.com","name":"Homes.com","category":"property","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Akamai"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":215,"bytes":365,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/homes.com.json"},{"schema":"ratradar.site/v1","host":"streeteasy.com","name":"StreetEasy","category":"property","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Enumerate pages from the sitemap: https://streeteasy.com/blog/sitemap_index.xml"],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront","PerimeterX"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"istio-envoy"},"robots":{"found":true,"disallow_all":false,"disallow_rules":32,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://streeteasy.com/blog/sitemap_index.xml","https://streeteasy.com/neighborhoods/sitemap_index.xml","https://streeteasy.com/guides/sitemap_index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://streeteasy.com/building/225-rector-place-new_york/sale/1836515","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://streeteasy.com/sitemapz/hdp/hdp_sales_0.xml.gz"}},"performance":{"response_ms":224,"bytes":353599,"redirects":0,"cookies_set":3},"why_this_score":[{"label":"CloudFront detected","points":2},{"label":"PerimeterX seen by the browser","points":20},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/streeteasy.com.json"},{"schema":"ratradar.site/v1","host":"www.apartmentlist.com","name":"Apartment List","category":"property","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.apartmentlist.com/sitemap.xml","Read the JSON-LD on the page (WebSite, Organization, Service). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.apartmentlist.com/api/trpc/locations.geolocate. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":"ApartmentList"},"robots":{"found":true,"disallow_all":false,"disallow_rules":8,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.apartmentlist.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebSite","Organization","Service"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.apartmentlist.com/api/trpc/locations.geolocate"],"inner_page":{"url":"https://www.apartmentlist.com/fl/osceola-county/welcome-home-to-2625-emerald-lake-ct-kissimmee-fl-34744","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.apartmentlist.com/sitemap-properties.xml"}},"performance":{"response_ms":184,"bytes":249128,"redirects":0,"cookies_set":6},"why_this_score":[{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/apartmentlist.com.json"},{"schema":"ratradar.site/v1","host":"www.rent.com","name":"Rent.com","category":"property","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Enumerate pages from the sitemap: https://www.rent.com/sitemap.xml","The site publishes 1 RSS or Atom feed. Prefer them for new items."],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["AWS WAF","CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"hidden"},"robots":{"found":true,"disallow_all":false,"disallow_rules":22,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.rent.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":1,"feed_urls":["https://www.rent.com/rss.xml"],"api_endpoints_seen":[],"inner_page":{"url":"https://www.rent.com/r/302-e-17th-st-bloomington-in-lv1318451938","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.rent.com/sitemap/Pdps/sitemap_0000.xml"}},"performance":{"response_ms":338,"bytes":327341,"redirects":0,"cookies_set":4},"why_this_score":[{"label":"AWS WAF detected","points":12},{"label":"CloudFront detected","points":2},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":429,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/rent.com.json"},{"schema":"ratradar.site/v1","host":"www.funda.nl","name":"Funda","category":"property","country":"Netherlands","verdict":"open","hostility_score":10,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"AkamaiNetStorage"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":75,"bytes":13906,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/funda.nl.json"},{"schema":"ratradar.site/v1","host":"www.immowelt.de","name":"Immowelt","category":"property","country":"Germany","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Enumerate pages from the sitemap: https://www.immowelt.de/sitemaps/sitemap_index.xml","Read the JSON-LD on the page (Organization). It is cleaner than the markup."],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["DataDome","CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"CloudFront"},"robots":{"found":true,"disallow_all":false,"disallow_rules":70,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.immowelt.de/sitemaps/sitemap_index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.immowelt.de/suche/mieten/hotel-gastro/schleswig-holstein/bad-schwartau-23611/ad08de388","status":200,"json_ld_types":["Organization","RealEstateListing","Product","BreadcrumbList"],"skipped":null,"sitemap":"https://www.immowelt.de/sitemaps/RENT_GASTRONOMY-HOTEL/rent_gastronomy-hotel_1.xml"}},"performance":{"response_ms":523,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"DataDome detected","points":20},{"label":"CloudFront detected","points":2},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/immowelt.de.json"},{"schema":"ratradar.site/v1","host":"www.immobiliare.it","name":"Immobiliare","category":"property","country":"Italy","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies.","Enumerate pages from the sitemap: https://www.immobiliare.it/sitemap.xml"],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["DataDome","Fastly"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":"DataDome"},"robots":{"found":true,"disallow_all":false,"disallow_rules":33,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.immobiliare.it/sitemap.xml","https://www.immobiliare.it/news/sitemap_index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":29,"bytes":774,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"DataDome detected","points":20},{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/immobiliare.it.json"},{"schema":"ratradar.site/v1","host":"www.fotocasa.es","name":"Fotocasa","category":"property","country":"Spain","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Read the JSON-LD on the page (WebSite). It is cleaner than the markup."],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":296,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":["WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":981,"bytes":400000,"redirects":1,"cookies_set":1},"why_this_score":[{"label":"CloudFront detected","points":2},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/fotocasa.es.json"},{"schema":"ratradar.site/v1","host":"www.daft.ie","name":"Daft.ie","category":"property","country":"Ireland","verdict":"blocked","hostility_score":76,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":79,"bytes":116688,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/daft.ie.json"},{"schema":"ratradar.site/v1","host":"www.propertyguru.com.sg","name":"PropertyGuru","category":"property","country":"Singapore","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":40,"bytes":5627,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/propertyguru.com.sg.json"},{"schema":"ratradar.site/v1","host":"www.magicbricks.com","name":"Magicbricks","category":"property","country":"India","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Enumerate pages from the sitemap: https://www.magicbricks.com/sitemap_index.xml"],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"newlan"},"robots":{"found":true,"disallow_all":false,"disallow_rules":20,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.magicbricks.com/sitemap_index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.magicbricks.com/news/hi/shah-rukh-khan-and-rizwan-sajan-launch-new-ultra-luxury-tower-shahrukhz-residences-in-dubai-jkmb/152109.html","status":200,"json_ld_types":["NewsArticle","WebPage","BreadcrumbList","Organization"],"skipped":null,"sitemap":"https://www.magicbricks.com/MBnews.xml"}},"performance":{"response_ms":723,"bytes":400000,"redirects":0,"cookies_set":3},"why_this_score":[{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/magicbricks.com.json"},{"schema":"ratradar.site/v1","host":"www.lamudi.com","name":"Lamudi","category":"property","country":"Philippines","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here."],"risks":["Hostile to automation. Keep volumes low and cache.","robots.txt forbids some AI crawlers: Bytespider, Amazonbot.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare","CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":44,"crawl_delay_seconds":null,"ai_crawlers_blocked":["Bytespider","Amazonbot"],"ai_crawlers_blocked_count":2,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":815,"bytes":40091,"redirects":0,"cookies_set":6},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"CloudFront detected","points":2},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 2 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/lamudi.com.json"},{"schema":"ratradar.site/v1","host":"www.bayut.com","name":"Bayut","category":"property","country":"United Arab Emirates","verdict":"blocked","hostility_score":78,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies.","Enumerate pages from the sitemap: https://www.bayut.com/sitemap_index.xml"],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["CloudFront","Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"CloudFront"},"robots":{"found":true,"disallow_all":false,"disallow_rules":231,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.bayut.com/sitemap_index.xml","https://www.bayut.com/ar/sitemap_index.xml","https://www.bayut.com/zh/sitemap_index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":72,"bytes":null,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"HTTP 401 on a plain request","points":70},{"label":"CloudFront detected","points":2},{"label":"Cloudflare seen by the browser","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":401,"headless_browser_status":503,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/bayut.com.json"},{"schema":"ratradar.site/v1","host":"www.property24.com","name":"Property24","category":"property","country":"South Africa","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","A browser load called JSON endpoints that look like data, for example ninja.data.olxcdn.com/config-property24-web.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":113,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.property24.com/sitemap/Index"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["ninja.data.olxcdn.com/config-property24-web.json","www.property24.com/autocomplete/propertiesgrouped","www.property24.com/page-init-data","js.monitor.azure.com/scripts/b/ai.config.1.cfg.json","laquesis.data.olxcdn.com/assign"],"inner_page":null},"performance":{"response_ms":2868,"bytes":127890,"redirects":0,"cookies_set":6},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/property24.com.json"},{"schema":"ratradar.site/v1","host":"www.simplyhired.com","name":"SimplyHired","category":"jobs","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":57,"bytes":5580,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/simplyhired.com.json"},{"schema":"ratradar.site/v1","host":"jooble.org","name":"Jooble","category":"jobs","country":"Ukraine","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies.","Enumerate pages from the sitemap: https://jooble.org/sitemap.xml"],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":53,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://jooble.org/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":236,"bytes":5575,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/jooble.org.json"},{"schema":"ratradar.site/v1","host":"www.adzuna.com","name":"Adzuna","category":"jobs","country":"United Kingdom","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Enumerate pages from the sitemap: https://www.adzuna.com/sitemap_index.jobs_US.xml"],"risks":["Hostile to automation. Keep volumes low and cache.","robots.txt forbids 11 of 14 known AI crawlers (GPTBot, ClaudeBot, Claude-Web, anthropic-ai, ...). An agent that identifies as an AI crawler should not fetch this site.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":36,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ClaudeBot","Claude-Web","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Bytespider","Amazonbot","Applebot-Extended","Meta-ExternalAgent"],"ai_crawlers_blocked_count":11,"sitemap_urls":["https://www.adzuna.com/sitemap_index.jobs_US.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.adzuna.com/company/salesupply/hiring","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.adzuna.com/sitemap-1.jobs_US.xml"}},"performance":{"response_ms":161,"bytes":79622,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"CloudFront detected","points":2},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 11 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/adzuna.com.json"},{"schema":"ratradar.site/v1","host":"www.totaljobs.com","name":"Totaljobs","category":"jobs","country":"United Kingdom","verdict":"guarded","hostility_score":45,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429.","A browser load called JSON endpoints that look like data, for example www.totaljobs.com/ul99iXkER/OEK/ZnR6qQ/tEGarbXzQGLGrVt1/PxVSYg/JAcEIy/RyaFU. Untested, check the terms before using them."],"risks":["robots.txt forbids some AI crawlers: Amazonbot."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":139,"crawl_delay_seconds":null,"ai_crawlers_blocked":["Amazonbot"],"ai_crawlers_blocked_count":1,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.totaljobs.com/ul99iXkER/OEK/ZnR6qQ/tEGarbXzQGLGrVt1/PxVSYg/JAcEIy/RyaFU"],"inner_page":null},"performance":{"response_ms":15051,"bytes":null,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"No answer to a plain request","points":40},{"label":"Slow response (15051 ms)","points":5},{"label":"A real browser got in though a plain request did not","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 1 AI crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":0,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/totaljobs.com.json"},{"schema":"ratradar.site/v1","host":"www.welcometothejungle.com","name":"Welcome to the Jungle","category":"jobs","country":"France","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here."],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":5,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.welcometothejungle.com/sitemaps/index.xml.gz"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":538,"bytes":400000,"redirects":1,"cookies_set":0},"why_this_score":[{"label":"CloudFront detected","points":2},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/welcometothejungle.com.json"},{"schema":"ratradar.site/v1","host":"www.xing.com","name":"Xing","category":"jobs","country":"Germany","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Read the JSON-LD on the page (WebSite). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.xing.com/graphql/api. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"CloudFront"},"robots":{"found":true,"disallow_all":false,"disallow_rules":222,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.xing.com/sitemap.xml.gz"]},"machine_readable":{"sitemap":false,"llms_txt":true,"json_ld_types":["WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.xing.com/graphql/api","v1.api.service.cmp.usercentrics.eu/latest/core/Tr_3B1GDkkB1q2","ingress.eu-west-1.aws.dash0.com/v1/traces","v1.api.service.cmp.usercentrics.eu/latest/1/cmp/en/TCF/Tr_3B1GDkkB1q2/449.140.599/449.140.599"],"inner_page":null},"performance":{"response_ms":654,"bytes":400000,"redirects":0,"cookies_set":3},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/xing.com.json"},{"schema":"ratradar.site/v1","host":"www.infojobs.net","name":"InfoJobs","category":"jobs","country":"Spain","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.infojobs.net/sitemap.xml","Read the JSON-LD on the page (WebSite). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.infojobs.net/ijlibrarym.js. Untested, check the terms before using them."],"risks":["robots.txt forbids 11 of 14 known AI crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"Apache"},"robots":{"found":true,"disallow_all":false,"disallow_rules":54,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","OAI-SearchBot","ClaudeBot","CCBot","PerplexityBot","Google-Extended","Bytespider","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":11,"sitemap_urls":[]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.infojobs.net/ijlibrarym.js"],"inner_page":{"url":"http://www.infojobs.net/informa/indicadores/2011/05","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.infojobs.net/sitemap.xml"}},"performance":{"response_ms":665,"bytes":51998,"redirects":0,"cookies_set":7},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 11 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":405,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/infojobs.net.json"},{"schema":"ratradar.site/v1","host":"www.naukrigulf.com","name":"Naukri Gulf","category":"jobs","country":"United Arab Emirates","verdict":"guarded","hostility_score":45,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":15042,"bytes":null,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"No answer to a plain request","points":40},{"label":"Slow response (15042 ms)","points":5}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":0,"headless_browser_status":0,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/naukrigulf.com.json"},{"schema":"ratradar.site/v1","host":"www.bayt.com","name":"Bayt","category":"jobs","country":"United Arab Emirates","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":25,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":true,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":32,"bytes":5552,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/bayt.com.json"},{"schema":"ratradar.site/v1","host":"www.brightermonday.co.ke","name":"BrighterMonday","category":"jobs","country":"Kenya","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.brightermonday.co.ke/sitemap-category-en.xml","The site publishes 2 RSS or Atom feeds. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example www.brightermonday.co.ke/ajax/home-search/search-suggestions. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":24,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.brightermonday.co.ke/sitemap-category-en.xml","https://www.brightermonday.co.ke/sitemap-directory-en.xml","https://www.brightermonday.co.ke/sitemap-listings-index-en.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":2,"feed_urls":["https://cde.hexagon.build/feeds/37xbMD?token=ab06a1a3-7aa7-4a8e-978e-4d2001718ef5","https://www.brightermonday.co.ke/rss-articles.xml"],"api_endpoints_seen":["www.brightermonday.co.ke/ajax/home-search/search-suggestions","www.brightermonday.co.ke/ajax/home-search/search-recents","www.brightermonday.co.ke/ajax/home-search/location-suggestions","www.brightermonday.co.ke/ajax/home-search/location-recents"],"inner_page":{"url":"https://www.brightermonday.co.ke/listings/sales-lead-b2b-salary-70000-80000-ksh-gross-n9ddvm","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.brightermonday.co.ke/sitemap-listings-sales-en.xml"}},"performance":{"response_ms":270,"bytes":400000,"redirects":0,"cookies_set":3},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/brightermonday.co.ke.json"},{"schema":"ratradar.site/v1","host":"builtin.com","name":"Built In","category":"jobs","country":"United States","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://builtin.com/sitemap.xml"],"risks":["robots.txt forbids 4 of 14 known AI crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, PerplexityBot). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":119,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","OAI-SearchBot","PerplexityBot"],"ai_crawlers_blocked_count":4,"sitemap_urls":["https://builtin.com/sitemap.xml","https://builtin.com/sitemaps.txt","https://builtin.com/sitemaps-1.txt"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://builtin.com/jobs/phoenix-mesa-scottsdale/sales/customer-success","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://builtin.com/job-board-sitemap.xml"}},"performance":{"response_ms":120,"bytes":68397,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 4 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/builtin.com.json"},{"schema":"ratradar.site/v1","host":"www.workingnomads.com","name":"Working Nomads","category":"jobs","country":"Portugal","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.workingnomads.com/sitemap.xml","A browser load called JSON endpoints that look like data, for example d262ri5vl0r1ym.cloudfront.net/jobs/data/new_jobs_today.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"nginx/1.18.0 (Ubuntu)"},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["d262ri5vl0r1ym.cloudfront.net/jobs/data/new_jobs_today.json","www.workingnomads.com/jobsapi/_search","www.workingnomads.com/api/preview-index/_search","cmp.inmobi.com/geoip","cmp.inmobi.com/GVL-v3/vendor-list-trimmed-v1.json"],"inner_page":{"url":"https://www.workingnomads.com/companies-hiring-remote-workers/fully-remote","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.workingnomads.com/sitemap.xml"}},"performance":{"response_ms":672,"bytes":86565,"redirects":1,"cookies_set":4},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/workingnomads.com.json"},{"schema":"ratradar.site/v1","host":"www.yummly.com","name":"Yummly","category":"food","country":"United States","verdict":"guarded","hostility_score":45,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":15315,"bytes":null,"redirects":1,"cookies_set":0},"why_this_score":[{"label":"No answer to a plain request","points":40},{"label":"Slow response (15315 ms)","points":5}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":0,"headless_browser_status":0,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/yummly.com.json"},{"schema":"ratradar.site/v1","host":"www.foodnetwork.com","name":"Food Network","category":"food","country":"United States","verdict":"open","hostility_score":10,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.foodnetwork.com/sitemaps/sitemap_food_index.xml","A browser load called JSON endpoints that look like data, for example buffet.identityservices.io/sso/api/1/check. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":11,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.foodnetwork.com/sitemaps/sitemap_food_index.xml","https://www.foodnetwork.com/fn-dish/news-sitemap.xml","https://www.foodnetwork.com/healthyeats/news-sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["buffet.identityservices.io/sso/api/1/check"],"inner_page":{"url":"https://www.foodnetwork.com/fn-dish/news/thanksgiving-butterball-report-2026","status":200,"json_ld_types":["Article","ItemList","BreadcrumbList"],"skipped":null,"sitemap":"https://www.foodnetwork.com/fn-dish/news-sitemap.xml"}},"performance":{"response_ms":230,"bytes":334378,"redirects":0,"cookies_set":4},"why_this_score":[{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/foodnetwork.com.json"},{"schema":"ratradar.site/v1","host":"www.seriouseats.com","name":"Serious Eats","category":"food","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Enumerate pages from the sitemap: https://www.seriouseats.com/sitemap.xml","Read the JSON-LD on the page (WebPage). It is cleaner than the markup.","The site publishes 1 RSS or Atom feed. Prefer them for new items."],"risks":["Hostile to automation. Keep volumes low and cache.","robots.txt forbids 11 of 14 known AI crawlers (ClaudeBot, Claude-Web, anthropic-ai, CCBot, ...). An agent that identifies as an AI crawler should not fetch this site.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":2,"crawl_delay_seconds":null,"ai_crawlers_blocked":["ClaudeBot","Claude-Web","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Bytespider","Amazonbot","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":11,"sitemap_urls":["https://www.seriouseats.com/sitemap.xml","https://www.seriouseats.com/google-news-sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebPage"],"feeds":1,"feed_urls":["https://feeds-api.dotdashmeredith.com/v1/rss/google/ad57d421-0ff5-41e7-a5da-f2167b6d7f7a"],"api_endpoints_seen":[],"inner_page":{"url":"https://www.seriouseats.com/bakery-style-desserts-recipes-12159606","status":200,"json_ld_types":["NewsArticle"],"skipped":null,"sitemap":"https://www.seriouseats.com/google-news-sitemap.xml"}},"performance":{"response_ms":248,"bytes":400000,"redirects":0,"cookies_set":5},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 11 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/seriouseats.com.json"},{"schema":"ratradar.site/v1","host":"www.bbcgoodfood.com","name":"BBC Good Food","category":"food","country":"United Kingdom","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.bbcgoodfood.com/sitemap.xml","Read the JSON-LD on the page (WebPage). It is cleaner than the markup.","The site publishes 2 RSS or Atom feeds. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example geoip.immediatemedia.co.uk/country-code. Untested, check the terms before using them."],"risks":["robots.txt forbids 4 of 14 known AI crawlers (GPTBot, CCBot, Bytespider, Meta-ExternalAgent). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":36,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","CCBot","Bytespider","Meta-ExternalAgent"],"ai_crawlers_blocked_count":4,"sitemap_urls":["https://www.bbcgoodfood.com/sitemap.xml","https://www.bbcgoodfood.com/google-news.xml","https://www.bbcgoodfood.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebPage"],"feeds":2,"feed_urls":["https://www.bbcgoodfood.com/feed","https://www.bbcgoodfood.com/feed/atom"],"api_endpoints_seen":["geoip.immediatemedia.co.uk/country-code","cp.bbcgoodfood.com/properties/791268da/config.json"],"inner_page":{"url":"https://www.bbcgoodfood.com/recipes/collection/esther-clarks-30-minute-comfort-meals","status":200,"json_ld_types":["Article","BreadcrumbList"],"skipped":null,"sitemap":"https://www.bbcgoodfood.com/google-news.xml"}},"performance":{"response_ms":63,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 4 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/bbcgoodfood.com.json"},{"schema":"ratradar.site/v1","host":"glovoapp.com","name":"Glovo","category":"food","country":"Spain","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","A browser load called JSON endpoints that look like data, for example api.glovoapp.com/features/query. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":3,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://glovoapp.com/sitemap-index.xml","https://glovoapp.com/nextjs-pages-sitemap.xml"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["api.glovoapp.com/features/query","v1.api.service.eu1.cmp.usercentrics.eu/latest/core/3l-LWoLH2QhZhI","v1.api.service.eu1.cmp.usercentrics.eu/latest/1/cmp/en/GDPR/3l-LWoLH2QhZhI/66.19.83/66.19.83"],"inner_page":null},"performance":{"response_ms":1372,"bytes":170328,"redirects":2,"cookies_set":1},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/glovoapp.com.json"},{"schema":"ratradar.site/v1","host":"wolt.com","name":"Wolt","category":"food","country":"Finland","verdict":"open","hostility_score":8,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://wolt.com/sitemap.xml","A browser load called JSON endpoints that look like data, for example wolt.com/v1/geo_ip_approximate. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront","Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"CloudFront"},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://wolt.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["wolt.com/v1/geo_ip_approximate","wolt.com/v1/country_list","wolt.com/regatta/v1/consumer_client/assignments","restaurant-api.wolt.com/v1/cities","restaurant-api.wolt.com/v2/config"],"inner_page":{"url":"https://wolt.com/fr/lux/luxembourg/venue/cactus-shoppi-howald/petit-pain-frais-90-g-itemid-b15767b437c7aa671cddadc8","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://wolt.com/sitemap/items/lux-1.xml"}},"performance":{"response_ms":823,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"CloudFront detected","points":2},{"label":"Cloudflare seen by the browser","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/wolt.com.json"},{"schema":"ratradar.site/v1","host":"www.lieferando.de","name":"Lieferando","category":"food","country":"Germany","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":171,"bytes":5685,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/lieferando.de.json"},{"schema":"ratradar.site/v1","host":"www.ifood.com.br","name":"iFood","category":"food","country":"Brazil","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies.","Enumerate pages from the sitemap: https://www.ifood.com.br/sitemap/sitemap.xml"],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":6,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.ifood.com.br/sitemap/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":183,"bytes":5556,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/ifood.com.br.json"},{"schema":"ratradar.site/v1","host":"www.talabat.com","name":"Talabat","category":"food","country":"United Arab Emirates","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.talabat.com/_sitemap/sitemap.xml.gz","A browser load called JSON endpoints that look like data, for example www.talabat.com/nextLocationApi/location/country-areas/4. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.talabat.com/_sitemap/sitemap.xml.gz","https://www.talabat.com/sitemap/sitemap.xml.gz"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.talabat.com/nextLocationApi/location/country-areas/4","ct.pinterest.com/user/"],"inner_page":{"url":"https://www.talabat.com/ar/uae/restaurants/103657/muraqqabat","status":200,"json_ld_types":["BreadcrumbList"],"skipped":null,"sitemap":"https://www.talabat.com/sitemap/uae/ar/areas.xml.gz"}},"performance":{"response_ms":408,"bytes":205839,"redirects":1,"cookies_set":14},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/talabat.com.json"},{"schema":"ratradar.site/v1","host":"www.gojek.com","name":"Gojek","category":"food","country":"Indonesia","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.gojek.com/sitemap.xml"],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":3,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.gojek.com/sitemap.xml","https://www.gojek.com/sitemap-portal-page.xml","https://www.gojek.com/sitemap-vn-blog.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.gojek.com/app/sdf-agreement","status":404,"json_ld_types":[],"skipped":null,"sitemap":"https://www.gojek.com/sitemap.xml"}},"performance":{"response_ms":2249,"bytes":101947,"redirects":1,"cookies_set":0},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/gojek.com.json"},{"schema":"ratradar.site/v1","host":"www.foodpanda.com","name":"Foodpanda","category":"food","country":"Singapore","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":42,"bytes":5642,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/foodpanda.com.json"},{"schema":"ratradar.site/v1","host":"data.gov.au","name":"data.gov.au","category":"gov","country":"Australia","verdict":"hostile","hostility_score":55,"recommended_approach":"respect_robots_skip","steps":["robots.txt disallows every crawler. Do not scrape it. Look for an official API, a data license, or a partner feed."],"risks":["Scraping against an explicit robots.txt ban is the quickest way to a legal or blocking problem.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":false,"official_api_advised":true},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"nginx"},"robots":{"found":true,"disallow_all":true,"disallow_rules":1,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":1037,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"CloudFront detected","points":2},{"label":"robots.txt disallows everything","points":10},{"label":"A headless browser was refused though a plain request worked","points":0},{"label":"Raised to at least 50 because robots.txt disallows everything","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt bans every crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/data.gov.au.json"},{"schema":"ratradar.site/v1","host":"data.gov.in","name":"data.gov.in","category":"gov","country":"India","verdict":"guarded","hostility_score":40,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":1840,"bytes":null,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"No answer to a plain request","points":40}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":0,"headless_browser_status":0,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/data.gov.in.json"},{"schema":"ratradar.site/v1","host":"open.canada.ca","name":"open.canada.ca","category":"gov","country":"Canada","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","robots.txt asks for a crawl delay of 20 seconds.","Enumerate pages from the sitemap: https://open.canada.ca/sitemap.xml","A browser load called JSON endpoints that look like data, for example open.canada.ca/invitation-manager/config.JSON. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":73,"crawl_delay_seconds":20,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["open.canada.ca/invitation-manager/config.JSON","open.canada.ca/invitation-manager/im.JSON"],"inner_page":{"url":"https://open.canada.ca/en/content/consultations/year1","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://open.canada.ca/sitemap.xml"}},"performance":{"response_ms":652,"bytes":23233,"redirects":1,"cookies_set":1},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/open.canada.ca.json"},{"schema":"ratradar.site/v1","host":"www.destatis.de","name":"Destatis","category":"gov","country":"Germany","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","robots.txt asks for a crawl delay of 30 seconds.","Enumerate pages from the sitemap: https://www.destatis.de/Sitemap_Index.xml","The site publishes 1 RSS or Atom feed. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example www.destatis.de/SiteGlobals/Modules/CookieBanner/DE/Allgemein/CookieBanner.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"myracloud"},"robots":{"found":true,"disallow_all":false,"disallow_rules":2,"crawl_delay_seconds":30,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.destatis.de/Sitemap_Index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":1,"feed_urls":["https://www.destatis.de/DE/Home/SiteGlobals/Functions/RSSFeed/DE/RSSNewsfeed/Aktuell.xml"],"api_endpoints_seen":["www.destatis.de/SiteGlobals/Modules/CookieBanner/DE/Allgemein/CookieBanner.json"],"inner_page":{"url":"https://www.destatis.de/DE/Methoden/Demografische-Regionale-Standards/_inhalt.html","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.destatis.de/Sitemap_Nodes.xml"}},"performance":{"response_ms":1774,"bytes":106128,"redirects":1,"cookies_set":0},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/destatis.de.json"},{"schema":"ratradar.site/v1","host":"www.insee.fr","name":"INSEE","category":"gov","country":"France","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.insee.fr/sitemap-index.xml","Read the JSON-LD on the page (WebSite). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example www.insee.fr/static/locales/fr/translation.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"unknown"},"robots":{"found":true,"disallow_all":false,"disallow_rules":6,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.insee.fr/sitemap-index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.insee.fr/static/locales/fr/translation.json"],"inner_page":null},"performance":{"response_ms":1911,"bytes":64391,"redirects":1,"cookies_set":1},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/insee.fr.json"},{"schema":"ratradar.site/v1","host":"www.istat.it","name":"Istat","category":"gov","country":"Italy","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.istat.it/sitemap.xml","The site publishes 1 RSS or Atom feed. Prefer them for new items."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":1,"feed_urls":["https://www.istat.it/feed/"],"api_endpoints_seen":[],"inner_page":{"url":"https://www.istat.it/en/news/the-migration-of-short-term-economic-statistics-to-the-new-ateco-2007-classification/","status":200,"json_ld_types":["BreadcrumbList"],"skipped":null,"sitemap":"https://www.istat.it/post-sitemap.xml"}},"performance":{"response_ms":2411,"bytes":278516,"redirects":0,"cookies_set":2},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/istat.it.json"},{"schema":"ratradar.site/v1","host":"www.federalreserve.gov","name":"Federal Reserve","category":"gov","country":"United States","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","A browser load called JSON endpoints that look like data, for example edge.api.brightcove.com/playback/v1/accounts/66043936001/videos/6405157968112. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["edge.api.brightcove.com/playback/v1/accounts/66043936001/videos/6405157968112","www.federalreserve.gov/json/unemployment.json","www.federalreserve.gov/json/gdp.json","www.federalreserve.gov/json/pce.json","www.federalreserve.gov/json/calendar.json"],"inner_page":null},"performance":{"response_ms":60,"bytes":106233,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/federalreserve.gov.json"},{"schema":"ratradar.site/v1","host":"fred.stlouisfed.org","name":"FRED","category":"gov","country":"United States","verdict":"open","hostility_score":10,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","robots.txt asks for a crawl delay of 1 seconds.","Read the JSON-LD on the page (DataCatalog). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example fred.stlouisfed.org/Tp9Bn-ka/J0s/380/7iVTaFmNfi/DuJaS20pXQpV8z/QEQ-AQ/HHE/MYjAHMAk. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"Apache"},"robots":{"found":true,"disallow_all":false,"disallow_rules":6,"crawl_delay_seconds":1,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":["DataCatalog"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["fred.stlouisfed.org/Tp9Bn-ka/J0s/380/7iVTaFmNfi/DuJaS20pXQpV8z/QEQ-AQ/HHE/MYjAHMAk"],"inner_page":null},"performance":{"response_ms":53,"bytes":58321,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/fred.stlouisfed.org.json"},{"schema":"ratradar.site/v1","host":"www.bea.gov","name":"BEA","category":"gov","country":"United States","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.bea.gov/sitemap.xml"],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":34,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.bea.gov/news/blog/2017-02-08/services-data-coming-faster-gdp","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.bea.gov/articles/sitemap.xml"}},"performance":{"response_ms":431,"bytes":59040,"redirects":0,"cookies_set":0},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/bea.gov.json"},{"schema":"ratradar.site/v1","host":"www.irs.gov","name":"IRS","category":"gov","country":"United States","verdict":"open","hostility_score":10,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.irs.gov/sitemap.xml","A browser load called JSON endpoints that look like data, for example www.irs.gov/omnibar-search-results-json-v2. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":82,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.irs.gov/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.irs.gov/omnibar-search-results-json-v2","resources.digital-cloud-gov.medallia.com/wdcgov/20103/onsite/onsiteData1790098801654.json"],"inner_page":{"url":"https://www.irs.gov/pub/irs-access/f13614c.zip","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.irs.gov/sitemap.xml?page=1"}},"performance":{"response_ms":69,"bytes":146722,"redirects":0,"cookies_set":3},"why_this_score":[{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/irs.gov.json"},{"schema":"ratradar.site/v1","host":"www.ftc.gov","name":"FTC","category":"gov","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","robots.txt asks for a crawl delay of 5 seconds."],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":37,"crawl_delay_seconds":5,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":505,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/ftc.gov.json"},{"schema":"ratradar.site/v1","host":"www.fcc.gov","name":"FCC","category":"gov","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Akamai"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"AkamaiGHost"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":33,"bytes":365,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/fcc.gov.json"},{"schema":"ratradar.site/v1","host":"www.faa.gov","name":"FAA","category":"gov","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Akamai"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Akamai","server":"AkamaiGHost"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":106,"bytes":365,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/faa.gov.json"},{"schema":"ratradar.site/v1","host":"www.ntsb.gov","name":"NTSB","category":"gov","country":"United States","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","A browser load called JSON endpoints that look like data, for example www.ntsb.gov/_api/SP.Web.GetContextWebThemeData. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":16,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.ntsb.gov/_api/SP.Web.GetContextWebThemeData","js.monitor.azure.com/scripts/b/ai.config.1.cfg.json"],"inner_page":null},"performance":{"response_ms":77,"bytes":65768,"redirects":1,"cookies_set":1},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/ntsb.gov.json"},{"schema":"ratradar.site/v1","host":"www.epa.gov","name":"EPA","category":"gov","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.epa.gov/sitemap.xml","A browser load called JSON endpoints that look like data, for example www.epa.gov/views/ajax. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":28,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.epa.gov/views/ajax"],"inner_page":{"url":"https://www.epa.gov/epcra/are-hazardous-chemicals-present-rail-yards-subject-epcra-311312","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.epa.gov/sitemap.xml?page=1"}},"performance":{"response_ms":106,"bytes":65052,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/epa.gov.json"},{"schema":"ratradar.site/v1","host":"reporter.nih.gov","name":"NIH Reporter","category":"gov","country":"United States","verdict":"open","hostility_score":15,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls.","A browser load called JSON endpoints that look like data, for example reporter.nih.gov/services/AppSettings/getSettingByName. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":["x-rate-limit-limit","x-rate-limit-remaining","x-rate-limit-reset"],"cdn":"","server":""},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["reporter.nih.gov/services/AppSettings/getSettingByName","reporter.nih.gov/services/Lookup/orgStates","reporter.nih.gov/services/Lookup/fiscalYears","reporter.nih.gov/services/Lookup/agencies","reporter.nih.gov/services/Chart/","reporter.nih.gov/data/geo-usa.json"],"inner_page":null},"performance":{"response_ms":295,"bytes":3339,"redirects":0,"cookies_set":4},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/reporter.nih.gov.json"},{"schema":"ratradar.site/v1","host":"medlineplus.gov","name":"MedlinePlus","category":"gov","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://medlineplus.gov/sitemap.xml","Read the JSON-LD on the page (WebSite, Organization). It is cleaner than the markup."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"AmazonS3"},"robots":{"found":true,"disallow_all":false,"disallow_rules":15,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://medlineplus.gov/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebSite","Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://medlineplus.gov/recipes/breakfast","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://medlineplus.gov/sitemap.xml"}},"performance":{"response_ms":105,"bytes":26483,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/medlineplus.gov.json"},{"schema":"ratradar.site/v1","host":"data.who.int","name":"WHO Data","category":"gov","country":"Switzerland","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: http://www.who.int/sitemaps/sitemapindex.xml"],"risks":["robots.txt forbids some AI crawlers: CCBot."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":2,"crawl_delay_seconds":null,"ai_crawlers_blocked":["CCBot"],"ai_crawlers_blocked_count":1,"sitemap_urls":["http://www.who.int/sitemaps/sitemapindex.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":230,"bytes":56137,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 1 AI crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/data.who.int.json"},{"schema":"ratradar.site/v1","host":"www.esa.int","name":"ESA","category":"gov","country":"France","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","robots.txt asks for a crawl delay of 2 seconds."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":20,"crawl_delay_seconds":2,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":219,"bytes":55922,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/esa.int.json"},{"schema":"ratradar.site/v1","host":"www.weather.gov","name":"NWS Weather","category":"gov","country":"United States","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"Apache"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":66,"bytes":128142,"redirects":0,"cookies_set":0},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/weather.gov.json"},{"schema":"ratradar.site/v1","host":"www.metoffice.gov.uk","name":"Met Office","category":"gov","country":"United Kingdom","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.metoffice.gov.uk/sitemap.xml","Read the JSON-LD on the page (Organization). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example apikeys.civiccomputing.com/c/v. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":13,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.metoffice.gov.uk/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["apikeys.civiccomputing.com/c/v","cc.cdn.civiccomputing.com/vl/v3/vendor-list.json"],"inner_page":{"url":"https://www.metoffice.gov.uk/research/news/2021/how-did-covid-19-lockdowns-affect-the-climate","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.metoffice.gov.uk/sitemap.xml"}},"performance":{"response_ms":246,"bytes":169214,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/metoffice.gov.uk.json"},{"schema":"ratradar.site/v1","host":"ourworldindata.org","name":"Our World in Data","category":"gov","country":"United Kingdom","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://ourworldindata.org/sitemap.xml","The site publishes 1 RSS or Atom feed. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example ourworldindata.org/dods.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://ourworldindata.org/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":1,"feed_urls":["https://ourworldindata.org/atom.xml"],"api_endpoints_seen":["ourworldindata.org/dods.json","ourworldindata.org/api/detect-country","ourworldindata.org/topicTagGraph.json"],"inner_page":{"url":"https://ourworldindata.org/data-insights/over-33000-near-earth-asteroids-have-been-detected-since-1990","status":200,"json_ld_types":["Article"],"skipped":null,"sitemap":"https://ourworldindata.org/sitemap.xml"}},"performance":{"response_ms":79,"bytes":263052,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/ourworldindata.org.json"},{"schema":"ratradar.site/v1","host":"www.nature.com","name":"Nature","category":"gov","country":"United Kingdom","verdict":"open","hostility_score":8,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.nature.com/sitemap.xml","The site publishes 1 RSS or Atom feed. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example idp.nature.com/exposed-details. Untested, check the terms before using them."],"risks":["robots.txt forbids 13 of 14 known AI crawlers (GPTBot, ChatGPT-User, ClaudeBot, Claude-Web, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly","Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":"Oscar Platform 0.2478.0"},"robots":{"found":true,"disallow_all":false,"disallow_rules":48,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","ClaudeBot","Claude-Web","anthropic-ai","CCBot","PerplexityBot","Google-Extended","Bytespider","Amazonbot","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":13,"sitemap_urls":["https://www.nature.com/sitemap.xml","https://www.nature.com/subjects/sitemap.xml","https://www.nature.com/naturecareers/sitemapindex.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":1,"feed_urls":["https://www.nature.com/nature.rss"],"api_endpoints_seen":["idp.nature.com/exposed-details","ep1.adtrafficquality.google/getconfig/sodar"],"inner_page":{"url":"https://www.nature.com/articles/d41586-026-03159-z","status":200,"json_ld_types":["WebPage"],"skipped":null,"sitemap":"https://www.nature.com/latest-news/sitemap.xml"}},"performance":{"response_ms":1668,"bytes":236686,"redirects":4,"cookies_set":1},"why_this_score":[{"label":"Fastly detected","points":2},{"label":"Cloudflare seen by the browser","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 13 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/nature.com.json"},{"schema":"ratradar.site/v1","host":"www.sciencedirect.com","name":"ScienceDirect","category":"edu","country":"Netherlands","verdict":"blocked","hostility_score":76,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":48,"bytes":400000,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/sciencedirect.com.json"},{"schema":"ratradar.site/v1","host":"www.ssrn.com","name":"SSRN","category":"edu","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","robots.txt asks for a crawl delay of 5 seconds."],"risks":["Hostile to automation. Keep volumes low and cache.","robots.txt forbids 3 of 14 known AI crawlers (GPTBot, ChatGPT-User, Google-Extended). An agent that identifies as an AI crawler should not fetch this site.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":3,"crawl_delay_seconds":5,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","Google-Extended"],"ai_crawlers_blocked_count":3,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":123,"bytes":70416,"redirects":2,"cookies_set":1},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 3 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/ssrn.com.json"},{"schema":"ratradar.site/v1","host":"www.biorxiv.org","name":"bioRxiv","category":"edu","country":"United States","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","robots.txt asks for a crawl delay of 7 seconds.","Enumerate pages from the sitemap: https://www.biorxiv.org/sitemap.xml"],"risks":["robots.txt forbids 7 of 14 known AI crawlers (GPTBot, ChatGPT-User, ClaudeBot, CCBot, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":37,"crawl_delay_seconds":7,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","ClaudeBot","CCBot","PerplexityBot","Bytespider","Amazonbot"],"ai_crawlers_blocked_count":7,"sitemap_urls":[]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.biorxiv.org/content/10.1101/036897v1","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.biorxiv.org/sitemap.xml?page=1"}},"performance":{"response_ms":45,"bytes":31830,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 7 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/biorxiv.org.json"},{"schema":"ratradar.site/v1","host":"scholar.google.com","name":"Google Scholar","category":"edu","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here."],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"scholar"},"robots":{"found":true,"disallow_all":false,"disallow_rules":7,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":395,"bytes":132154,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":429,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/scholar.google.com.json"},{"schema":"ratradar.site/v1","host":"openalex.org","name":"OpenAlex","category":"edu","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls."],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":106,"bytes":4968,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15},{"label":"Cloudflare detected","points":6},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/openalex.org.json"},{"schema":"ratradar.site/v1","host":"www.crossref.org","name":"Crossref","category":"edu","country":"United Kingdom","verdict":"open","hostility_score":8,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.crossref.org/sitemap.xml"],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront","Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"AmazonS3"},"robots":{"found":true,"disallow_all":false,"disallow_rules":3,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.crossref.org/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.crossref.org/news/2010-12-06-kisti-registers-44000-crossref-dois-for-korean-scholarly-papers/","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.crossref.org/sitemap.xml"}},"performance":{"response_ms":312,"bytes":35282,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"CloudFront detected","points":2},{"label":"Cloudflare seen by the browser","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/crossref.org.json"},{"schema":"ratradar.site/v1","host":"www.trustradius.com","name":"TrustRadius","category":"reviews","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":46,"bytes":5580,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/trustradius.com.json"},{"schema":"ratradar.site/v1","host":"www.consumerreports.org","name":"Consumer Reports","category":"reviews","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Enumerate pages from the sitemap: https://www.consumerreports.org/sitemap.xml","Read the JSON-LD on the page (WebSite, Organization). It is cleaner than the markup."],"risks":["Hostile to automation. Keep volumes low and cache.","robots.txt forbids 3 of 14 known AI crawlers (GPTBot, CCBot, Google-Extended). An agent that identifies as an AI crawler should not fetch this site.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":13,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","CCBot","Google-Extended"],"ai_crawlers_blocked_count":3,"sitemap_urls":[]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebSite","Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.consumerreports.org/products/sitemap/28939","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.consumerreports.org/sitemaps/products.xml"}},"performance":{"response_ms":212,"bytes":400000,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"CloudFront detected","points":2},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 3 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/consumerreports.org.json"},{"schema":"ratradar.site/v1","host":"www.which.co.uk","name":"Which","category":"reviews","country":"United Kingdom","verdict":"open","hostility_score":4,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.which.co.uk/sitemap.xml","Read the JSON-LD on the page (WebPage, WebSite, ItemList). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example api.core-navigation-service-prod.product.which.co.uk/nav-data-prod/footer.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly","CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"Apache"},"robots":{"found":true,"disallow_all":false,"disallow_rules":30,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.which.co.uk/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebPage","WebSite","ItemList"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["api.core-navigation-service-prod.product.which.co.uk/nav-data-prod/footer.json","api.core-navigation-service-prod.product.which.co.uk/nav-data-prod/nav.json"],"inner_page":{"url":"https://www.which.co.uk/news/article/hargreaves-lansdown-launches-biggest-ever-cashback-offer-is-it-worth-it-aOwyP9R8nrbO","status":200,"json_ld_types":["BreadcrumbList","NewsArticle"],"skipped":null,"sitemap":"https://www.which.co.uk/news/sitemaps/news.xml"}},"performance":{"response_ms":1297,"bytes":400000,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"Fastly detected","points":2},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/which.co.uk.json"},{"schema":"ratradar.site/v1","host":"www.tripadvisor.co.uk","name":"Tripadvisor UK","category":"reviews","country":"United Kingdom","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Enumerate pages from the sitemap: https://www.tripadvisor.co.uk/sitemap/2/en_UK/sitemap_en_UK_index.xml","Read the JSON-LD on the page (Organization, WebSite, LocalBusiness). It is cleaner than the markup."],"risks":["Hostile to automation. Keep volumes low and cache.","robots.txt forbids 9 of 14 known AI crawlers (GPTBot, ClaudeBot, CCBot, Google-Extended, ...). An agent that identifies as an AI crawler should not fetch this site.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai","DataDome","CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":650,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ClaudeBot","CCBot","Google-Extended","Bytespider","Amazonbot","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":9,"sitemap_urls":["https://www.tripadvisor.co.uk/sitemap/2/en_UK/sitemap_en_UK_index.xml","https://www.tripadvisor.co.uk/sitemap/2/en_UK/sitemap_en_UK_location_photo_direct_link_index.xml","https://www.tripadvisor.co.uk/sitemap/2/en_UK/sitemap_en_UK_show_user_reviews_index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["Organization","WebSite","LocalBusiness"],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://www.tripadvisor.co.uk/AttractionProductReview-g255100-d11457667-Phillip_Island_Penguin_Parade_Day_Trip_with_Koalas_from_Melbourne-Melbourne_Victor.html","status":200,"json_ld_types":["Organization","WebSite","BreadcrumbList","Product","WebPage"],"skipped":null,"sitemap":"https://www.tripadvisor.co.uk/sitemap/att/en_UK/sitemap-2200191-en_UK-attraction_product_review-1791378804.xml.gz"}},"performance":{"response_ms":568,"bytes":400000,"redirects":0,"cookies_set":4},"why_this_score":[{"label":"Akamai detected","points":10},{"label":"DataDome detected","points":20},{"label":"CloudFront detected","points":2},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 9 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/tripadvisor.co.uk.json"},{"schema":"ratradar.site/v1","host":"www.homeadvisor.com","name":"HomeAdvisor","category":"reviews","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans.","robots.txt forbids some AI crawlers: GPTBot."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":23,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot"],"ai_crawlers_blocked_count":1,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":32,"bytes":5580,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"strong","signals":[{"kind":"robots_txt","label":"robots.txt blocks 1 AI crawler"},{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/homeadvisor.com.json"},{"schema":"ratradar.site/v1","host":"steamcommunity.com","name":"Steam Community","category":"social","country":"United States","verdict":"open","hostility_score":10,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Akamai"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":5,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":100,"bytes":47585,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"Akamai detected","points":10}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/steamcommunity.com.json"},{"schema":"ratradar.site/v1","host":"www.roblox.com","name":"Roblox","category":"media","country":"United States","verdict":"guarded","hostility_score":45,"recommended_approach":"plain_http_slow","steps":["Plain HTTP works but is watched. Keep to about one request per second, send a real user agent, and back off on 429.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls.","Enumerate pages from the sitemap: https://www.roblox.com/sitemap.xml","Read the JSON-LD on the page (Organization). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example apis.roblox.com/guac-v2/v1/bundles/footer-ui. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare","PerimeterX","Fastly","CloudFront"],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":["x-ratelimit-limit","x-ratelimit-remaining","x-ratelimit-reset"],"cdn":"","server":"public-gateway"},"robots":{"found":true,"disallow_all":false,"disallow_rules":28,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.roblox.com/sitemap.xml","https://www.roblox.com/sitemap-games.xml","https://www.roblox.com/sitemap-ar.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["apis.roblox.com/guac-v2/v1/bundles/footer-ui","apis.roblox.com/guac-v2/v1/bundles/cookie-policy","locale.roblox.com/v1/locales","assetdelivery.roblox.com/v2/asset"],"inner_page":{"url":"https://www.roblox.com/charts/top-trending","status":200,"json_ld_types":["CollectionPage","FAQPage"],"skipped":null,"sitemap":"https://www.roblox.com/sitemap.xml"}},"performance":{"response_ms":129,"bytes":100147,"redirects":0,"cookies_set":3},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15},{"label":"Cloudflare detected","points":6},{"label":"PerimeterX detected","points":20},{"label":"Fastly detected","points":2},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/roblox.com.json"},{"schema":"ratradar.site/v1","host":"imgur.com","name":"Imgur","category":"social","country":"United States","verdict":"open","hostility_score":17,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls.","A browser load called JSON endpoints that look like data, for example api.imgur.com/account/v1/battle/user. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Fastly"],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"Fastly","server":"cat factory 1.0"},"robots":{"found":true,"disallow_all":false,"disallow_rules":9,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["api.imgur.com/account/v1/battle/user","api.imgur.com/3/configuration/desktop","api.imgur.com/homepage/v1/messages/random","api.imgur.com/homepage/v1/cover_photos/random","api.imgur.com/3/tags"],"inner_page":null},"performance":{"response_ms":24,"bytes":6675,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15},{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/imgur.com.json"},{"schema":"ratradar.site/v1","host":"www.deviantart.com","name":"DeviantArt","category":"social","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","robots.txt asks for a crawl delay of 1 seconds.","Read the JSON-LD on the page (Organization, WebSite, WebPage). It is cleaner than the markup."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":11,"crawl_delay_seconds":1,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.deviantart.com/sitemap-index.xml.gz"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":["Organization","WebSite","WebPage"],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":556,"bytes":400000,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/deviantart.com.json"},{"schema":"ratradar.site/v1","host":"www.flickr.com","name":"Flickr","category":"social","country":"United States","verdict":"hostile","hostility_score":50,"recommended_approach":"respect_robots_skip","steps":["robots.txt disallows every crawler. Do not scrape it. Look for an official API, a data license, or a partner feed.","A browser load called JSON endpoints that look like data, for example cdn.weglot.com/projects-settings/0a57b843dd2eb8584ade033db89f74c87.json. Untested, check the terms before using them."],"risks":["Scraping against an explicit robots.txt ban is the quickest way to a legal or blocking problem."],"can_scrape":{"plain_http":false,"headless_browser":true,"needs_proxies":false,"official_api_advised":true},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":""},"robots":{"found":true,"disallow_all":true,"disallow_rules":1,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.flickr.com/sitemap/index/users/sitemap-index-users-00000000.xml.gz","https://www.flickr.com/sitemap/index/tags/sitemap-index-tags-00000000.xml.gz","https://www.flickr.com/sitemap/index/sets/sitemap-index-sets-00000000.xml.gz"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["cdn.weglot.com/projects-settings/0a57b843dd2eb8584ade033db89f74c87.json"],"inner_page":null},"performance":{"response_ms":119,"bytes":58542,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"CloudFront detected","points":2},{"label":"robots.txt disallows everything","points":10},{"label":"Raised to at least 50 because robots.txt disallows everything","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt bans every crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/flickr.com.json"},{"schema":"ratradar.site/v1","host":"500px.com","name":"500px","category":"social","country":"Canada","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","A browser load called JSON endpoints that look like data, for example api-neo.500px.com/graphql. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"AliyunOSS"},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["api-neo.500px.com/graphql"],"inner_page":null},"performance":{"response_ms":85,"bytes":3337,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/500px.com.json"},{"schema":"ratradar.site/v1","host":"www.behance.net","name":"Behance","category":"social","country":"United States","verdict":"hostile","hostility_score":72,"recommended_approach":"real_browser","steps":["Use a rendering browser, slow requests, and rotating proxies.","Enumerate pages from the sitemap: https://www.behance.net/sitemap"],"risks":["Hostile to automation. Keep volumes low and cache.","robots.txt forbids 9 of 14 known AI crawlers (GPTBot, ClaudeBot, Claude-Web, anthropic-ai, ...). An agent that identifies as an AI crawler should not fetch this site."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":false},"protections":{"vendors":["Fastly"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":["retry-after"],"cdn":"Fastly","server":"Varnish"},"robots":{"found":true,"disallow_all":false,"disallow_rules":24,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ClaudeBot","Claude-Web","anthropic-ai","CCBot","Google-Extended","Bytespider","Applebot-Extended","Meta-ExternalAgent"],"ai_crawlers_blocked_count":9,"sitemap_urls":["https://www.behance.net/sitemap"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":30,"bytes":453,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 9 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/behance.net.json"},{"schema":"ratradar.site/v1","host":"dribbble.com","name":"Dribbble","category":"social","country":"United States","verdict":"hostile","hostility_score":59,"recommended_approach":"real_browser","steps":["Use a rendering browser, slow requests, and rotating proxies.","Enumerate pages from the sitemap: https://dribbble.com/sitemap.xml"],"risks":["Hostile to automation. Keep volumes low and cache."],"can_scrape":{"plain_http":false,"headless_browser":true,"needs_proxies":true,"official_api_advised":false},"protections":{"vendors":["AWS WAF","CloudFront"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"awselb/2.0"},"robots":{"found":true,"disallow_all":false,"disallow_rules":38,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://dribbble.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://dribbble.com/jobs/183719-Graphic-Designer","status":200,"json_ld_types":["JobPosting"],"skipped":null,"sitemap":"https://dribbble.com/sitemap/graphic_design_jobs.xml"}},"performance":{"response_ms":204,"bytes":2013,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"A challenge page appeared","points":45},{"label":"AWS WAF detected","points":12},{"label":"CloudFront detected","points":2},{"label":"A real browser got in though a plain request did not","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":202,"headless_browser_status":202,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/dribbble.com.json"},{"schema":"ratradar.site/v1","host":"app.thestorygraph.com","name":"StoryGraph","category":"reviews","country":"United Kingdom","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":44,"bytes":5561,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/app.thestorygraph.com.json"},{"schema":"ratradar.site/v1","host":"www.fandom.com","name":"Fandom","category":"social","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"official_api_or_skip","steps":["Both a plain request and a headless browser were refused. Look for an official API or a data partner before spending on proxies."],"risks":["Active bot protection. Expect blocks, CAPTCHAs, and account or IP bans."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":true,"official_api_advised":true},"protections":{"vendors":["Cloudflare","Fastly"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":39,"bytes":5575,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6},{"label":"Fastly detected","points":2}],"eu_ai_training":{"level":"some","signals":[{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/fandom.com.json"},{"schema":"ratradar.site/v1","host":"tvtropes.org","name":"TV Tropes","category":"media","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","The site publishes 3 RSS or Atom feeds. Prefer them for new items."],"risks":["Hostile to automation. Keep volumes low and cache.","robots.txt forbids 11 of 14 known AI crawlers (GPTBot, ChatGPT-User, ClaudeBot, anthropic-ai, ...). An agent that identifies as an AI crawler should not fetch this site.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":["GPTBot","ChatGPT-User","ClaudeBot","anthropic-ai","CCBot","PerplexityBot","Bytespider","Amazonbot","Applebot-Extended","cohere-ai","Meta-ExternalAgent"],"ai_crawlers_blocked_count":11,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":3,"feed_urls":["https://tvtropes.org/pmwiki/rss-feed.php?filter=newest_pages","https://tvtropes.org/pmwiki/rss-feed.php?filter=most_popular","https://tvtropes.org/pmwiki/rss-feed.php?filter=updated_content"],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":201,"bytes":134003,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 11 AI crawlers"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/tvtropes.org.json"},{"schema":"ratradar.site/v1","host":"slashdot.org","name":"Slashdot","category":"social","country":"United States","verdict":"open","hostility_score":8,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://slashdot.org/sitemap-slashdot-index.xml","The site publishes 1 RSS or Atom feed. Prefer them for new items."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare","CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":85,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://slashdot.org/sitemap-slashdot-index.xml","https://apache.slashdot.org/sitemap-apache-index.xml","https://apple.slashdot.org/sitemap-apple-index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":[],"feeds":1,"feed_urls":["https://rss.slashdot.org/Slashdot/slashdotMain"],"api_endpoints_seen":[],"inner_page":{"url":"https://news.slashdot.org/story/26/10/11/0142235/ubuntu-confirms-thursdays-distributed-denial-of-service-attack-against-its-web-infrastructure","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://news.slashdot.org/news_sitemap-news.xml"}},"performance":{"response_ms":74,"bytes":151825,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/slashdot.org.json"},{"schema":"ratradar.site/v1","host":"lobste.rs","name":"Lobsters","category":"social","country":"United States","verdict":"hostile","hostility_score":50,"recommended_approach":"respect_robots_skip","steps":["robots.txt disallows every crawler. Do not scrape it. Look for an official API, a data license, or a partner feed.","robots.txt asks for a crawl delay of 1 seconds.","The site publishes 2 RSS or Atom feeds. Prefer them for new items."],"risks":["Scraping against an explicit robots.txt ban is the quickest way to a legal or blocking problem."],"can_scrape":{"plain_http":false,"headless_browser":true,"needs_proxies":false,"official_api_advised":true},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":true,"disallow_rules":1,"crawl_delay_seconds":1,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://lobste.rs/sitemap.xml.gz"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":2,"feed_urls":["https://lobste.rs/rss","https://lobste.rs/comments.rss"],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":444,"bytes":61195,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"robots.txt disallows everything","points":10},{"label":"Raised to at least 50 because robots.txt disallows everything","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt bans every crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/lobste.rs.json"},{"schema":"ratradar.site/v1","host":"truthsocial.com","name":"Truth Social","category":"social","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here."],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":59,"bytes":3496,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/truthsocial.com.json"},{"schema":"ratradar.site/v1","host":"gab.com","name":"Gab","category":"social","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Enumerate pages from the sitemap: https://gab.com/sitemap.xml","Read the JSON-LD on the page (WebSite, Organization, WebPage, BreadcrumbList). It is cleaner than the markup."],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":25,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://gab.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebSite","Organization","WebPage","BreadcrumbList","FAQPage"],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://gab.com/marketplace/listings/apparel","status":200,"json_ld_types":["WebSite","Organization","CollectionPage","FAQPage","BreadcrumbList"],"skipped":null,"sitemap":"https://gab.com/sitemap.xml"}},"performance":{"response_ms":128,"bytes":113349,"redirects":0,"cookies_set":3},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/gab.com.json"},{"schema":"ratradar.site/v1","host":"www.xiaohongshu.com","name":"Xiaohongshu","category":"social","country":"China","verdict":"hostile","hostility_score":50,"recommended_approach":"respect_robots_skip","steps":["robots.txt disallows every crawler. Do not scrape it. Look for an official API, a data license, or a partner feed.","Enumerate pages from the sitemap: https://www.xiaohongshu.com/sitemap.xml","A browser load called JSON endpoints that look like data, for example pages.xiaohongshu.com/data/sem_sdk. Untested, check the terms before using them."],"risks":["Scraping against an explicit robots.txt ban is the quickest way to a legal or blocking problem."],"can_scrape":{"plain_http":false,"headless_browser":true,"needs_proxies":false,"official_api_advised":true},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":true,"login_wall":true,"rate_limit_headers":[],"cdn":"","server":"TencentEdgeOne"},"robots":{"found":true,"disallow_all":true,"disallow_rules":1,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.xiaohongshu.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["pages.xiaohongshu.com/data/sem_sdk","as.xiaohongshu.com/api/sec/v1/sbtsource","as.xiaohongshu.com/api/p/pj","as.xiaohongshu.com/api/sec/v1/scripting","pages.xiaohongshu.com/data/xhs_pc_nps"],"inner_page":null},"performance":{"response_ms":872,"bytes":36679,"redirects":2,"cookies_set":2},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15},{"label":"robots.txt disallows everything","points":10},{"label":"Raised to at least 50 because robots.txt disallows everything","points":0}],"eu_ai_training":{"level":"strong","signals":[{"kind":"robots_txt","label":"robots.txt bans every crawler"},{"kind":"login_wall","label":"a login wall"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/xiaohongshu.com.json"},{"schema":"ratradar.site/v1","host":"www.zhihu.com","name":"Zhihu","category":"social","country":"China","verdict":"hostile","hostility_score":60,"recommended_approach":"respect_robots_skip","steps":["robots.txt disallows every crawler. Do not scrape it. Look for an official API, a data license, or a partner feed.","A browser load called JSON endpoints that look like data, for example www.zhihu.com/api/v3/oauth/sms/supported_countries. Untested, check the terms before using them."],"risks":["Scraping against an explicit robots.txt ban is the quickest way to a legal or blocking problem.","robots.txt forbids some AI crawlers: Google-Extended."],"can_scrape":{"plain_http":false,"headless_browser":true,"needs_proxies":false,"official_api_advised":true},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"BLB/25.12.0.2"},"robots":{"found":true,"disallow_all":true,"disallow_rules":1,"crawl_delay_seconds":null,"ai_crawlers_blocked":["Google-Extended"],"ai_crawlers_blocked_count":1,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.zhihu.com/api/v3/oauth/sms/supported_countries","www.zhihu.com/zbst/events/r"],"inner_page":null},"performance":{"response_ms":249,"bytes":214,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"robots.txt disallows everything","points":10},{"label":"A real browser got in though a plain request did not","points":0},{"label":"Raised to at least 50 because robots.txt disallows everything","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt bans every crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/zhihu.com.json"},{"schema":"ratradar.site/v1","host":"www.daum.net","name":"Daum","category":"search","country":"South Korea","verdict":"hostile","hostility_score":50,"recommended_approach":"respect_robots_skip","steps":["robots.txt disallows every crawler. Do not scrape it. Look for an official API, a data license, or a partner feed.","A browser load called JSON endpoints that look like data, for example dapi.kakao.com/region-code/v2/regionWithWeather.json. Untested, check the terms before using them."],"risks":["Scraping against an explicit robots.txt ban is the quickest way to a legal or blocking problem."],"can_scrape":{"plain_http":false,"headless_browser":true,"needs_proxies":false,"official_api_advised":true},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":true,"disallow_rules":1,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["dapi.kakao.com/region-code/v2/regionWithWeather.json","ads-api.daum.net/api/v2/ads","ads-api.daum.net/api/v1/shopping/naver","shop-redapi.daum.net/red-apigate/p/pctop/plans"],"inner_page":null},"performance":{"response_ms":1840,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"robots.txt disallows everything","points":10},{"label":"Raised to at least 50 because robots.txt disallows everything","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt bans every crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/daum.net.json"},{"schema":"ratradar.site/v1","host":"hatena.ne.jp","name":"Hatena","category":"social","country":"Japan","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","A browser load called JSON endpoints that look like data, for example cdn.api.btloader.com/device. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"nginx"},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["cdn.api.btloader.com/device"],"inner_page":null},"performance":{"response_ms":2213,"bytes":400000,"redirects":2,"cookies_set":1},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/hatena.ne.jp.json"},{"schema":"ratradar.site/v1","host":"www.together.ai","name":"Together AI","category":"ai","country":"United States","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://docs.together.ai/sitemap.xml","Read the JSON-LD on the page (Organization, WebSite). It is cleaner than the markup.","The site publishes 1 RSS or Atom feed. Prefer them for new items.","A browser load called JSON endpoints that look like data, for example api.intellimize.co/context-v2/117655561. Untested, check the terms before using them."],"risks":["robots.txt forbids some AI crawlers: Google-Extended."],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":["Google-Extended"],"ai_crawlers_blocked_count":1,"sitemap_urls":["https://docs.together.ai/sitemap.xml","https://www.together.ai/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["Organization","WebSite"],"feeds":1,"feed_urls":["https://www.together.ai/blog/rss.xml"],"api_endpoints_seen":["api.intellimize.co/context-v2/117655561","api.intellimize.co/prediction/117655561"],"inner_page":{"url":"https://docs.together.ai/docs/inference/videos/audio-input","status":200,"json_ld_types":["WebSite","Organization","WebPage","BreadcrumbList","Article","TechArticle"],"skipped":null,"sitemap":"https://docs.together.ai/sitemap.xml"}},"performance":{"response_ms":73,"bytes":400000,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt blocks 1 AI crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/together.ai.json"},{"schema":"ratradar.site/v1","host":"groq.com","name":"Groq","category":"ai","country":"United States","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://groq.com/sitemap.xml","A browser load called JSON endpoints that look like data, for example groq.com/a/flags. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Vercel","server":"Vercel"},"robots":{"found":true,"disallow_all":false,"disallow_rules":2,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://groq.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["groq.com/a/flags"],"inner_page":{"url":"https://groq.com/newsroom/groq-named-a-top-10-national-security-tech-company-as-it-scales-u-s-built-ai-infrastructure","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://groq.com/sitemap.xml"}},"performance":{"response_ms":202,"bytes":55004,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/groq.com.json"},{"schema":"ratradar.site/v1","host":"fireworks.ai","name":"Fireworks AI","category":"ai","country":"United States","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://fireworks.ai/sitemap.xml","Read the JSON-LD on the page (Organization, WebSite). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example cdn.dreamdata.cloud/api/v1/p. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Vercel","server":"Vercel"},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://fireworks.ai/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["Organization","WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["cdn.dreamdata.cloud/api/v1/p"],"inner_page":{"url":"https://fireworks.ai/events/microsoft-build-2026","status":200,"json_ld_types":["Organization","WebSite"],"skipped":null,"sitemap":"https://fireworks.ai/sitemap.xml"}},"performance":{"response_ms":493,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Cloudflare seen by the browser","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/fireworks.ai.json"},{"schema":"ratradar.site/v1","host":"openrouter.ai","name":"OpenRouter","category":"ai","country":"United States","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://openrouter.ai/sitemap.xml","Read the JSON-LD on the page (Organization, WebSite). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example clerk.openrouter.ai/v1/environment. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":1,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://openrouter.ai/sitemap.xml","https://openrouter.ai/blog/sitemap-index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["Organization","WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["clerk.openrouter.ai/v1/environment","clerk.openrouter.ai/v1/client","openrouter.ai/api/frontend/v1/newsletter/capture-units"],"inner_page":{"url":"https://openrouter.ai/apps/category/coding/cli-agent","status":200,"json_ld_types":["Organization","WebSite","BreadcrumbList","CollectionPage","ItemList"],"skipped":null,"sitemap":"https://openrouter.ai/sitemaps/apps.xml"}},"performance":{"response_ms":108,"bytes":261835,"redirects":0,"cookies_set":4},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/openrouter.ai.json"},{"schema":"ratradar.site/v1","host":"stability.ai","name":"Stability AI","category":"ai","country":"United Kingdom","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://stability.ai/sitemap.xml","Read the JSON-LD on the page (WebSite). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example forms-eu1.hsforms.com/embed/v3/form/26117460/296f89b5-3f8a-4eab-add9-5153aa2cc2b4/json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"Squarespace"},"robots":{"found":true,"disallow_all":false,"disallow_rules":27,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://stability.ai/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["forms-eu1.hsforms.com/embed/v3/form/26117460/296f89b5-3f8a-4eab-add9-5153aa2cc2b4/json","stability.ai/api/census/RecordHit"],"inner_page":{"url":"https://stability.ai/news-updates/category/Company","status":200,"json_ld_types":["WebSite"],"skipped":null,"sitemap":"https://stability.ai/sitemap.xml"}},"performance":{"response_ms":304,"bytes":400000,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"Cloudflare seen by the browser","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/stability.ai.json"},{"schema":"ratradar.site/v1","host":"runwayml.com","name":"Runway","category":"ai","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://runway.com/sitemap.xml","Read the JSON-LD on the page (Organization, SoftwareApplication, WebSite). It is cleaner than the markup."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Vercel","server":"Vercel"},"robots":{"found":true,"disallow_all":false,"disallow_rules":1,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://runway.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["Organization","SoftwareApplication","WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":156,"bytes":400000,"redirects":1,"cookies_set":4},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/runwayml.com.json"},{"schema":"ratradar.site/v1","host":"elevenlabs.io","name":"ElevenLabs","category":"ai","country":"United States","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://elevenlabs.io/sitemap.xml","Read the JSON-LD on the page (Organization, WebSite, Brand, WebPage). It is cleaner than the markup."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"nginx/1.30.4"},"robots":{"found":true,"disallow_all":false,"disallow_rules":18,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://elevenlabs.io/sitemap.xml","https://help.elevenlabs.io/hc/sitemap.xml","https://elevenlabs.io/docs/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["Organization","WebSite","Brand","WebPage","BreadcrumbList","SoftwareApplication"],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://elevenlabs.io/sv/careers/813f6b98-fa14-4d8f-902d-5d09029be20a/account-executive-enterprise","status":200,"json_ld_types":["Organization","WebSite","Brand","JobPosting"],"skipped":null,"sitemap":"https://elevenlabs.io/sitemap/jobs__sv.xml"}},"performance":{"response_ms":134,"bytes":400000,"redirects":0,"cookies_set":3},"why_this_score":[{"label":"Cloudflare seen by the browser","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/elevenlabs.io.json"},{"schema":"ratradar.site/v1","host":"x.ai","name":"xAI","category":"ai","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","Enumerate pages from the sitemap: https://x.ai/sitemap.xml","Read the JSON-LD on the page (WebSite, Organization). It is cleaner than the markup."],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":1,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://x.ai/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebSite","Organization"],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://x.ai/news/grok-4-7","status":200,"json_ld_types":["NewsArticle"],"skipped":null,"sitemap":"https://x.ai/sitemap.xml"}},"performance":{"response_ms":119,"bytes":248090,"redirects":0,"cookies_set":3},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/x.ai.json"},{"schema":"ratradar.site/v1","host":"www.deepseek.com","name":"DeepSeek","category":"ai","country":"China","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://www.deepseek.com/sitemap.xml","A browser load called JSON endpoints that look like data, for example chat.deepseek.com/api/v0/ip_to_country_code. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"AmazonS3"},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.deepseek.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["chat.deepseek.com/api/v0/ip_to_country_code","gator.volces.com/webid","gator.volces.com/list"],"inner_page":{"url":"https://www.deepseek.com/en/news/deepseek-v4-1-flash/","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.deepseek.com/sitemap.xml"}},"performance":{"response_ms":40,"bytes":115583,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/deepseek.com.json"},{"schema":"ratradar.site/v1","host":"www.kimi.com","name":"Kimi","category":"ai","country":"China","verdict":"open","hostility_score":21,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls.","Enumerate pages from the sitemap: https://www.kimi.com/sitemap.xml","A browser load called JSON endpoints that look like data, for example gator.volces.com/webid. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":5,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://www.kimi.com/sitemap.xml","https://www.kimi.com/sitemap/sitemap-ug.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["gator.volces.com/webid","apmplus.volces.com/settings/get/webpro","www.kimi.com/update.json"],"inner_page":{"url":"https://www.kimi.com/membership/pricing","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://www.kimi.com/sitemap.xml"}},"performance":{"response_ms":633,"bytes":139867,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15},{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/kimi.com.json"},{"schema":"ratradar.site/v1","host":"qwen.ai","name":"Qwen","category":"ai","country":"China","verdict":"open","hostility_score":15,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls.","A browser load called JSON endpoints that look like data, for example qwen.ai/api/config. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":false,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["qwen.ai/api/config","qwen.ai/api/page_config","qwen.ai/api/v2/article/retrieval","alidt.alicdn.com/alilog/configs/sdk/common.json"],"inner_page":null},"performance":{"response_ms":2205,"bytes":94344,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/qwen.ai.json"},{"schema":"ratradar.site/v1","host":"character.ai","name":"Character.AI","category":"ai","country":"United States","verdict":"hostile","hostility_score":55,"recommended_approach":"plain_http","steps":["Use plain HTTP requests. Headless browsers are fingerprinted and refused here.","The home page is a JavaScript shell, so raw HTML is mostly empty. Use a rendering browser or the JSON endpoints the page calls.","Enumerate pages from the sitemap: https://character.ai/sitemap_index.xml"],"risks":["Hostile to automation. Keep volumes low and cache.","Headless browsers are refused here, so a browser-based actor will fail."],"can_scrape":{"plain_http":true,"headless_browser":false,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":true,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":5,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://character.ai/sitemap_index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://character.ai/character/HD2nZncl/young-aemeath","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://character.ai/sitemaps/sitemap_1.xml"}},"performance":{"response_ms":300,"bytes":331019,"redirects":0,"cookies_set":6},"why_this_score":[{"label":"Home page is a JavaScript shell","points":15},{"label":"Cloudflare detected","points":6},{"label":"A headless browser was refused though a plain request worked","points":0}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/character.ai.json"},{"schema":"ratradar.site/v1","host":"poe.com","name":"Poe","category":"ai","country":"United States","verdict":"blocked","hostility_score":100,"recommended_approach":"respect_robots_skip","steps":["robots.txt disallows every crawler. Do not scrape it. Look for an official API, a data license, or a partner feed."],"risks":["Scraping against an explicit robots.txt ban is the quickest way to a legal or blocking problem."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":false,"official_api_advised":true},"protections":{"vendors":["Cloudflare"],"challenge_page":true,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":true,"disallow_rules":14,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":141,"bytes":5671,"redirects":1,"cookies_set":1},"why_this_score":[{"label":"HTTP 403 on a plain request","points":70},{"label":"A challenge page appeared","points":25},{"label":"Cloudflare detected","points":6},{"label":"robots.txt disallows everything","points":10},{"label":"Raised to at least 50 because robots.txt disallows everything","points":0}],"eu_ai_training":{"level":"strong","signals":[{"kind":"robots_txt","label":"robots.txt bans every crawler"},{"kind":"captcha","label":"a CAPTCHA or bot challenge"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":403,"headless_browser_status":403,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/poe.com.json"},{"schema":"ratradar.site/v1","host":"airtable.com","name":"Airtable","category":"saas","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://airtable.com/sitemap.xml","A browser load called JSON endpoints that look like data, for example transcend-cdn.com/cm-test/619e6e3b-1a5c-4516-be11-6d77bdcbd717/uiV2/translations/us-template-v01/en.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"Vercel"},"robots":{"found":true,"disallow_all":false,"disallow_rules":17,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://airtable.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["transcend-cdn.com/cm-test/619e6e3b-1a5c-4516-be11-6d77bdcbd717/uiV2/translations/us-template-v01/en.json"],"inner_page":{"url":"https://www.airtable.com/lp/product/feature-prioritization","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://airtable.com/sitemap.xml"}},"performance":{"response_ms":470,"bytes":381347,"redirects":1,"cookies_set":8},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/airtable.com.json"},{"schema":"ratradar.site/v1","host":"zapier.com","name":"Zapier","category":"saas","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://zapier.com/uag6x9zaeydnbfy75xsggcsb37h8cgzf-two-service-sitemap.xml","Read the JSON-LD on the page (Organization, WebSite, WebPage, SoftwareApplication). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example zapier.com/api/v4/identity/. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"Vercel"},"robots":{"found":true,"disallow_all":false,"disallow_rules":24,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://zapier.com/uag6x9zaeydnbfy75xsggcsb37h8cgzf-two-service-sitemap.xml","https://zapier.com/uag6x9zaeydnbfy75xsggcsb37h8cgzf-one-service-sitemap.xml","https://zapier.com/sitemaps/zap-templates/LZB2XiWzbJFggRYdnRGwcoAgEkTnHHEh"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["Organization","WebSite","WebPage","SoftwareApplication"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["zapier.com/api/v4/identity/"],"inner_page":{"url":"https://zapier.com/apps/schedule/integrations/10000ft","status":200,"json_ld_types":["Organization","SoftwareApplication","Service","BreadcrumbList","VideoObject","HowTo"],"skipped":null,"sitemap":"https://zapier.com/uag6x9zaeydnbfy75xsggcsb37h8cgzf-two-service-sitemap-apps-two-service.xml"}},"performance":{"response_ms":141,"bytes":399652,"redirects":0,"cookies_set":2},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/zapier.com.json"},{"schema":"ratradar.site/v1","host":"linear.app","name":"Linear","category":"saas","country":"United States","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://linear.app/sitemap.xml","A browser load called JSON endpoints that look like data, for example e.linear.app/flags/. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":2,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://linear.app/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["e.linear.app/flags/"],"inner_page":{"url":"https://linear.app/careers/d3bc1ced-3ce4-4086-a050-555055dbb1ff","status":200,"json_ld_types":[],"skipped":null,"sitemap":"https://linear.app/sitemap.xml"}},"performance":{"response_ms":259,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/linear.app.json"},{"schema":"ratradar.site/v1","host":"asana.com","name":"Asana","category":"saas","country":"United States","verdict":"open","hostility_score":8,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://asana.com/sitemap-index.xml","Read the JSON-LD on the page (Organization, WebSite, BreadcrumbList). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example app.asana.com/-/marketing_info. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare","CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Netlify","server":"Netlify"},"robots":{"found":true,"disallow_all":false,"disallow_rules":14,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://asana.com/sitemap-index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["Organization","WebSite","BreadcrumbList"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["app.asana.com/-/marketing_info","api.zaius.com/v3/graphql"],"inner_page":{"url":"https://asana.com/jobs/apply","status":500,"json_ld_types":[],"skipped":null,"sitemap":"https://asana.com/sitemap.xml"}},"performance":{"response_ms":120,"bytes":400000,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/asana.com.json"},{"schema":"ratradar.site/v1","host":"trello.com","name":"Trello","category":"saas","country":"United States","verdict":"open","hostility_score":2,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","A browser load called JSON endpoints that look like data, for example api.atlassian.com/flags/api/v2/configurations. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"CloudFront","server":"AtlassianEdge"},"robots":{"found":true,"disallow_all":false,"disallow_rules":23,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://a594014.sitemaphosting7.com/4704043/sitemap_4704043.xml"]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["api.atlassian.com/flags/api/v2/configurations","api.atlassian.com/flags/api/v2/frontend/clientSdkKey/atl-global-static_web","api.atlassian.com/flags/api/v2/frontend/experimentValues","xp.atlassian.com/v1/rgstr","www.atlassian.com/gateway/api/graphql"],"inner_page":null},"performance":{"response_ms":129,"bytes":400000,"redirects":0,"cookies_set":0},"why_this_score":[{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/trello.com.json"},{"schema":"ratradar.site/v1","host":"monday.com","name":"monday.com","category":"saas","country":"Israel","verdict":"open","hostility_score":8,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://monday.com/sitemap_index.xml","Read the JSON-LD on the page (Organization, SoftwareApplication, WebSite, WebPage). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example vi.monday.com/api/create. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare","CloudFront"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":21,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://monday.com/sitemap_index.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["Organization","SoftwareApplication","WebSite","WebPage"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["vi.monday.com/api/create"],"inner_page":{"url":"https://monday.com/marketplace/listing/10000008/hootsuite","status":null,"json_ld_types":[],"skipped":"robots.txt disallows this path","sitemap":"https://monday.com/sitemap.xml"}},"performance":{"response_ms":244,"bytes":400000,"redirects":0,"cookies_set":5},"why_this_score":[{"label":"Cloudflare detected","points":6},{"label":"CloudFront detected","points":2}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/monday.com.json"},{"schema":"ratradar.site/v1","host":"you.com","name":"You.com","category":"search","country":"United States","verdict":"open","hostility_score":6,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://you.com/sitemap.xml","Read the JSON-LD on the page (Organization, WebSite). It is cleaner than the markup.","A browser load called JSON endpoints that look like data, for example cmp.osano.com/169xOaUXWTojE4kD7/63063ed3-cdab-4c42-90b0-7c61bf6b7dee/en.json. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["Cloudflare"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"Cloudflare","server":"cloudflare"},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":["https://you.com/sitemap.xml"]},"machine_readable":{"sitemap":true,"llms_txt":true,"json_ld_types":["Organization","WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":["cmp.osano.com/169xOaUXWTojE4kD7/63063ed3-cdab-4c42-90b0-7c61bf6b7dee/en.json"],"inner_page":{"url":"https://you.com/docs/additional-resources/changelog/2026/10/7","status":200,"json_ld_types":["BreadcrumbList","TechArticle"],"skipped":null,"sitemap":"https://you.com/sitemap.xml"}},"performance":{"response_ms":196,"bytes":393159,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"Cloudflare detected","points":6}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/you.com.json"},{"schema":"ratradar.site/v1","host":"kagi.com","name":"Kagi","category":"search","country":"United States","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","Enumerate pages from the sitemap: https://kagi.com/sitemap.xml","Read the JSON-LD on the page (WebSite). It is cleaner than the markup."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":""},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":true,"llms_txt":false,"json_ld_types":["WebSite"],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":{"url":"https://kagi.com/about","status":200,"json_ld_types":["WebSite"],"skipped":null,"sitemap":"https://kagi.com/sitemap.xml"}},"performance":{"response_ms":142,"bytes":67790,"redirects":0,"cookies_set":0},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/kagi.com.json"},{"schema":"ratradar.site/v1","host":"www.startpage.com","name":"Startpage","category":"search","country":"Netherlands","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"nginx"},"robots":{"found":true,"disallow_all":false,"disallow_rules":4,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":[],"inner_page":null},"performance":{"response_ms":442,"bytes":225227,"redirects":0,"cookies_set":0},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/startpage.com.json"},{"schema":"ratradar.site/v1","host":"www.qwant.com","name":"Qwant","category":"search","country":"France","verdict":"open","hostility_score":20,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","A browser load called JSON endpoints that look like data, for example dd.qwant.com/js/. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":["DataDome"],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"istio-envoy"},"robots":{"found":true,"disallow_all":false,"disallow_rules":4,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["dd.qwant.com/js/","api.qwant.com/v3/homepage/events/skin","mn.qwant.com/v2/api/feedback","mn.qwant.com/v2/api/ux/surveys"],"inner_page":null},"performance":{"response_ms":1074,"bytes":145558,"redirects":0,"cookies_set":1},"why_this_score":[{"label":"DataDome detected","points":20}],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/qwant.com.json"},{"schema":"ratradar.site/v1","host":"www.seznam.cz","name":"Seznam","category":"search","country":"Czechia","verdict":"open","hostility_score":0,"recommended_approach":"plain_http","steps":["A plain HTTP request gets the page. Still keep it polite: identify yourself and respect robots.txt.","A browser load called JSON endpoints that look like data, for example login.seznam.cz/api/v1/user/badge. Untested, check the terms before using them."],"risks":[],"can_scrape":{"plain_http":true,"headless_browser":true,"needs_proxies":false,"official_api_advised":false},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":["x-ratelimit-limit","x-ratelimit-remaining","x-ratelimit-reset"],"cdn":"","server":"envoy"},"robots":{"found":true,"disallow_all":false,"disallow_rules":0,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["login.seznam.cz/api/v1/user/badge","www.seznam.cz/api/v2/graphql","cmp.seznam.cz/arbiter/api/v1/cmp-configuration","login.seznam.cz/api/v1/autologin","d53-a.sdn.cz/d_53/sl/player_config/1.0/config.json","h.seznam.cz/udid"],"inner_page":null},"performance":{"response_ms":2485,"bytes":667021,"redirects":2,"cookies_set":4},"why_this_score":[],"eu_ai_training":{"level":"none","signals":[],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"None of the signals the draft names were seen. Personal data still falls under the GDPR: publicly available is not freely usable."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/seznam.cz.json"},{"schema":"ratradar.site/v1","host":"www.sogou.com","name":"Sogou","category":"search","country":"China","verdict":"hostile","hostility_score":50,"recommended_approach":"respect_robots_skip","steps":["robots.txt disallows every crawler. Do not scrape it. Look for an official API, a data license, or a partner feed.","A browser load called JSON endpoints that look like data, for example www.sogou.com/websearch/api/getcity. Untested, check the terms before using them."],"risks":["Scraping against an explicit robots.txt ban is the quickest way to a legal or blocking problem."],"can_scrape":{"plain_http":false,"headless_browser":false,"needs_proxies":false,"official_api_advised":true},"protections":{"vendors":[],"challenge_page":false,"javascript_shell":false,"login_wall":false,"rate_limit_headers":[],"cdn":"","server":"nginx"},"robots":{"found":true,"disallow_all":true,"disallow_rules":1,"crawl_delay_seconds":null,"ai_crawlers_blocked":[],"ai_crawlers_blocked_count":0,"sitemap_urls":[]},"machine_readable":{"sitemap":false,"llms_txt":false,"json_ld_types":[],"feeds":0,"feed_urls":[],"api_endpoints_seen":["www.sogou.com/websearch/api/getcity","hotlist.imtt.qq.com/GetRefreshInterval","trace.inlong.qq.com/atta2nd/dataproxy/message","hotlistad.html5.qq.com/trpc.tsbs_growth.hotwords.HotWords/GetHotWords","hotlist.imtt.qq.com/Fetch","hotlistad.html5.qq.com/trpc.tsbs_growth.hotwords.HotWords/ReportHotWords"],"inner_page":null},"performance":{"response_ms":710,"bytes":22764,"redirects":0,"cookies_set":4},"why_this_score":[{"label":"robots.txt disallows everything","points":10},{"label":"Raised to at least 50 because robots.txt disallows everything","points":0}],"eu_ai_training":{"level":"some","signals":[{"kind":"robots_txt","label":"robots.txt bans every crawler"}],"ai_txt_checked":false,"basis":"EDPB draft Guidelines 03/2026 on web scraping for generative AI (consultation open until 30 October 2026)","note":"The site signals that it does not expect to be scraped. Under the draft, that weighs against collecting personal data here to train generative AI. Not legal advice."},"ratlabs_actors":[],"probe":{"plain_http_status":200,"headless_browser_status":200,"vantage":"one residential machine, one day","note":"Heuristic. A site can behave differently by country, IP type, or after many requests. Respect robots.txt and the site terms."},"file":"https://ratlabs.tech/radar/api/v1/site/sogou.com.json"}]}