# sailing-whitsundays.com robots.txt
# Generated: 07 April 2026
# Safe for: Google Places, Google Ads, TikTok Business, Instagram / Meta / Facebook ads & link previews, GEO visibility

User-agent: *
Allow: /
Disallow: /booking/
Disallow: /checkout/
Disallow: /cart/
Disallow: /my-account/
Disallow: /order-confirmation/
Disallow: /thank-you/
Disallow: /payment/
Disallow: /admin/
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /*?*sort=*
Disallow: /*?*filter=*
Disallow: /*?*page=*
Disallow: /search/
Disallow: /tag/
Disallow: /category/*page/*
Disallow: /feed/
Disallow: /trackback/
Disallow: /comments/feed/
Disallow: /wp-json/
Disallow: /webapiwrapper/
Disallow: /product/get-price/

# ── Google — MUST ALLOW full crawl for Places & Ads ─────────────────
User-agent: Googlebot
Allow: /
Disallow: /thank-you/
Disallow: /booking/
Disallow: /checkout/

User-agent: Googlebot-Image
Allow: /
Disallow: /thank-you/
Disallow: /booking/
Disallow: /checkout/

User-agent: AdsBot-Google
Allow: /
Disallow: /thank-you/
Disallow: /booking/
Disallow: /checkout/
Crawl-delay: 5

# ── Meta / Facebook / Instagram — MUST ALLOW for ads, previews, quality score ──
User-agent: facebookexternalhit
Allow: /
Disallow: /thank-you/
Disallow: /booking/
Disallow: /checkout/
Crawl-delay: 5

User-agent: meta-externalads
Allow: /
Disallow: /thank-you/
Disallow: /booking/
Disallow: /checkout/
Crawl-delay: 5

# Block Meta AI training crawlers (no benefit for ads or visibility)
User-agent: meta-externalagent
Disallow: /

User-agent: Facebot
Disallow: /

User-agent: FacebookBot
Disallow: /

# ── TikTok Business / Ads / Pixel — MUST ALLOW ──────────────────────
User-agent: TikTokSpider
Allow: /
Disallow: /thank-you/
Disallow: /booking/
Disallow: /checkout/
Crawl-delay: 5

User-agent: ByteDance
Allow: /
Disallow: /thank-you/
Disallow: /booking/
Disallow: /checkout/
Crawl-delay: 5

User-agent: bytespider-ads
Allow: /
Disallow: /thank-you/
Disallow: /booking/
Disallow: /checkout/
Crawl-delay: 5

# Block aggressive TikTok/Douyin content scraper (separate from ads/pixel)
User-agent: Bytespider
Disallow: /

# ── Other good search engines ───────────────────────────────────────
User-agent: bingbot
Allow: /
Disallow: /thank-you/
Disallow: /booking/
Disallow: /checkout/
Crawl-delay: 5

User-agent: DuckDuckBot
Allow: /
Disallow: /thank-you/
Disallow: /booking/
Disallow: /checkout/
Crawl-delay: 5

User-agent: Applebot
Allow: /
Disallow: /thank-you/
Disallow: /booking/
Disallow: /checkout/
Crawl-delay: 10

User-agent: YandexBot
Allow: /
Disallow: /thank-you/
Disallow: /booking/
Disallow: /checkout/
Crawl-delay: 10

# ── Allow ChatGPT live browsing & search (critical for GEO) ────────
User-agent: ChatGPT-User
Allow: /whitsundays/
Allow: /tours/
Allow: /tour/
Allow: /article/
Allow: /articles/
Allow: /fraserisland/
Crawl-delay: 10

User-agent: OAI-SearchBot
Allow: /whitsundays/
Allow: /tours/
Allow: /tour/
Allow: /article/
Allow: /articles/
Allow: /fraserisland/
Crawl-delay: 10

# ── Block AI training & aggressive scrapers ─────────────────────────
User-agent: GPTBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Diffbot
Disallow: /

# ── SEO tools — throttle ────────────────────────────────────────────
User-agent: AhrefsBot
Crawl-delay: 30

User-agent: SemrushBot
Crawl-delay: 30

# ── Default for all other bots ──────────────────────────────────────
User-agent: *
Crawl-delay: 10

Sitemap: https://sailing-whitsundays.com/sitemap.xml