# https://auditro.co/robots.txt
#
# Auditro is a lead-generation site for a UAE ICV product. Being read, quoted
# and cited by answer engines is the point, so retrieval and search crawlers
# are allowed everywhere. The brand-internal files under this origin are not
# blocked here on purpose: a Disallow would stop crawlers reading the
# X-Robots-Tag: noindex header that vercel.json sends on them, which is what
# actually keeps them out of an index.

User-agent: *
Allow: /

# --- Answer engines and AI search --------------------------------------
# Retrieval and citation crawlers. These are the surfaces that send traffic.

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: MistralAI-User
Allow: /

User-agent: cohere-ai
Allow: /

# --- Bulk scrapers with no citation path -------------------------------
# These harvest at volume and return no referral or attribution. Excluded
# on cost-benefit, not principle; revisit if any starts citing sources.

User-agent: Bytespider
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: Omgili
Disallow: /

Sitemap: https://auditro.co/sitemap.xml