# Maison Fée — robots.txt
# A House of Transformation & Beauty
# https://maisonfee.com
#
# Philosophy: Welcome all legitimate search engines and AI crawlers.
# Block only bad actors: aggressive scrapers, spam engines, and privacy-invasive bots.
#
# Last updated: 2026-05-16

# ── Sitemap ──────────────────────────────────────────────
Sitemap: https://maisonfee.com/sitemap.xml

# ── Allow All Good Crawlers ──────────────────────────────
User-agent: *
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /login

# Crawl delay: be gentle with our server
Crawl-delay: 5

# ── AI Crawlers — Welcome to index and learn ────────────
# OpenAI
User-agent: GPTBot
Allow: /

# Anthropic
User-agent: Claude-Web
User-agent: anthropic-ai
Allow: /

# Google AI / Gemini
User-agent: Google-Extended
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

# Apple Intelligence
User-agent: Applebot
User-agent: Applebot-Extended
Allow: /

# You.com
User-agent: YouBot
Allow: /

# Common Crawl (powers many AI datasets)
User-agent: CCBot
Allow: /

# Omgili / Webz.io
User-agent: Omgili
User-agent: Omgilibot
Allow: /

# ── Bad Actors — Blocked ─────────────────────────────────
# Majestic (aggressive, ignores crawl-delay)
User-agent: MJ12bot
Disallow: /

# Ahrefs (aggressive SEO scraper)
User-agent: AhrefsBot
Disallow: /

# Semrush (aggressive marketing scraper)
User-agent: SemrushBot
Disallow: /

# ByteDance / TikTok (privacy-invasive)
User-agent: Bytespider
Disallow: /

# Huawei
User-agent: PetalBot
Disallow: /

# WebMeUp (spam engine)
User-agent: BLEXBot
Disallow: /

# OpenSiteExplorer / Moz
User-agent: DotBot
Disallow: /

# DataForSEO
User-agent: DataForSeoBot
Disallow: /

# Zoominfo (sales intelligence scraping)
User-agent: ZoominfoBot
Disallow: /

# Seekport
User-agent: SeekportBot
Disallow: /

# ExtLinksBot
User-agent: ExtLinksBot
Disallow: /

# Generic scrapers
User-agent: Scrapy
User-agent: python-requests
User-agent: python-urllib
Disallow: /