{
 "published": true,
 "published_at": "2026-11-02",
 "site": "https://lab.crawlcheck.io",
 "article": "https://crawlcheck.io/research/best-ai-audit-tool",
 "note": "A test site built by CrawlCheck. Nothing here is a real business. Each fault below is planted on purpose.",
 "faults": [
  {
   "id": "F1",
   "fault": "ClaudeBot is fully allowed",
   "detail": "robots.txt has User-agent: * with Disallow: /, but ClaudeBot has its own group holding only Crawl-delay: 5. Under RFC 9309 the * rules do not apply to it, so ClaudeBot may crawl everything.",
   "correct_answer": "ClaudeBot allowed, by its own group"
  },
  {
   "id": "F2",
   "fault": "Every unnamed crawler is blocked",
   "detail": "User-agent: * / Disallow: / shuts out every crawler without its own group (ChatGPT-User, Claude-SearchBot, Googlebot, Bingbot and others).",
   "correct_answer": "crawlers without a named group are disallowed"
  },
  {
   "id": "F3",
   "fault": "PerplexityBot is refused at the server",
   "detail": "robots.txt allows PerplexityBot; the server answers its user agent with HTTP 403.",
   "correct_answer": "403 for PerplexityBot despite robots.txt allowing it"
  },
  {
   "id": "F4",
   "fault": "Applebot gets a challenge page",
   "detail": "robots.txt allows Applebot; the server answers 200 with a 'Just a moment...' browser-check page instead of the content.",
   "correct_answer": "challenge page served to Applebot"
  },
  {
   "id": "F5",
   "fault": "OAI-SearchBot gets a stripped page",
   "detail": "OAI-SearchBot receives HTTP 200 with about 60% of the text removed compared with an ordinary client.",
   "correct_answer": "OAI-SearchBot served materially less text"
  },
  {
   "id": "F6",
   "fault": "Prices exist only after JavaScript",
   "detail": "The HTML says 'Loading prices...'; an inline script writes the real prices into #prices.",
   "correct_answer": "prices missing for crawlers that do not run scripts"
  },
  {
   "id": "F7",
   "fault": "llms.txt is malformed",
   "detail": "The file exists but has no H1 title on the first line and no markdown links.",
   "correct_answer": "llms.txt does not follow the format"
  },
  {
   "id": "F8",
   "fault": "The JSON-LD does not parse",
   "detail": "A trailing comma makes the LocalBusiness block invalid JSON.",
   "correct_answer": "JSON-LD invalid (parse error), not merely absent"
  },
  {
   "id": "F9",
   "fault": "security.txt has expired",
   "detail": "Expires: 2025-01-01T00:00:00Z, in the past (RFC 9116).",
   "correct_answer": "security.txt expired"
  },
  {
   "id": "F10",
   "fault": "The sitemap named in robots.txt is missing",
   "detail": "robots.txt declares Sitemap: https://lab.crawlcheck.io/sitemap.xml, which returns 404.",
   "correct_answer": "declared sitemap returns 404"
  }
 ]
}