# robots.txt for Quetext.com
# Purpose: Control access for web crawlers and LLM bots
# Last updated: 15 October 2025

# --- SEO and Audit Tools ---
User-agent: AhrefsSiteAudit
Allow: /

User-agent: AhrefsBot
Allow: /

User-agent: SemrushBot
Disallow: /

User-agent: proximic
Disallow: /


# --- General Bot Rules ---
User-agent: *
Allow: /assets/img/logos/*
Allow: /assets/img/illustrations/logo.svg
Disallow: /api/*
Disallow: /dist/*
Disallow: /results/*
Disallow: /report/
Disallow: /generate-citation
Disallow: /two-factor-login/
Disallow: /account/billing

# --- LLM and AI Crawlers ---
# Restrict model training and data ingestion
User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: OmgiliBot
Disallow: /

User-agent: FacebookBot
Disallow: /

# --- Notes ---
# Public pages (e.g., blog and resources) remain crawlable for SEO.
# AI crawlers must also respect /llm.txt for detailed usage rights.
# Contact: legal@quetext.com

Sitemap: https://www.quetext.com/sitemap.xml