# ---------------------------------

# firstforwomen Insurance robots.txt

# https://www.firstforwomen.co.za/

# Updated for LLM and Search Engine Harmony

# ---------------------------------


# Main Sitemaps

Sitemap: https://www.firstforwomen.co.za/sitemap.xml


# ---------------------------------

# Search Engine Crawlers

# ---------------------------------

User-agent: googlebot

Disallow:


User-agent: bingbot

Disallow:


User-agent: AdsBot-Google

Disallow: /dashboard*

Disallow: /activate*

Disallow: /link*

Disallow: /cdn-cgi/rum*


# ---------------------------------

# LLM / AI Crawlers (for visibility & summarization)

# ---------------------------------

User-agent: GPTBot

Allow: /


User-agent: ClaudeBot

Allow: /


User-agent: Google-Extended

Allow: /


User-agent: PerplexityBot

Allow: /


User-agent: Applebot

Allow: /


# Optional: Allow Microsoft Copilot / Edge AI experiences

User-agent: MSAIbot

Allow: /


# ---------------------------------

# Restricted / Sensitive Areas

# ---------------------------------

Disallow: /quote/

Disallow: /claims/

Disallow: /my-account/

Disallow: /login/

Disallow: /checkout/

Disallow: /payment/

Disallow: /admin/

Disallow: /api/

Disallow: /external-links

Disallow: /ab-tracking


# ---------------------------------

# Restricted AI Crawlers (aligned with llm.txt)

# ---------------------------------

User-agent: CCBot

Disallow: /


User-agent: MetaBot

Disallow: /


User-agent: Amazonbot

Disallow: /


# ---------------------------------

# Default Rule

# ---------------------------------

User-agent: *

Allow: /

Disallow: /external-links

Disallow: /ab-tracking