# ── AI / LLM crawlers — explicitly welcomed (GEO) ────────────────── # Explicit Allow signals stronger crawl intent than the implicit "*" rule. # These engines read our content for AI Overviews / answer citations. User-agent: GPTBot Allow: / User-agent: OAI-SearchBot Allow: / User-agent: ChatGPT-User Allow: / User-agent: ClaudeBot Allow: / User-agent: Claude-Web Allow: / User-agent: anthropic-ai Allow: / User-agent: Google-Extended Allow: / User-agent: PerplexityBot Allow: / User-agent: Applebot-Extended Allow: / User-agent: CCBot Allow: / User-agent: * Allow: / Allow: /library Allow: /blog Allow: /contact Allow: /about Allow: /upgrade # UGC / AI-creators product pages — explicit Allow (landing, catalog, details). # The /book questionnaire is a funnel step and is Disallowed below. Allow: /UGC-creator Allow: /UGC-creators # Locale-prefixed public pages — explicit Allow lines so Google won't get confused # by any path-style Disallow rules later. Vercel serves the same SPA shell for all. Allow: /en Allow: /ca Allow: /fr Allow: /tr # Legal pages (/privacy, /terms, /cookies, /legal): do NOT Disallow here. # If blocked, Google cannot crawl the page and will not see noindex — URLs can stay indexed. # Those URLs use noindex + X-Robots-Tag instead (see vercel.json + usePageTitle). # Block authenticated/user pages and confirmation/intermediate flow pages Disallow: /auth Disallow: /my-library Disallow: /settings Disallow: /restaurant-images Disallow: /restaurant-profile Disallow: /upload-dish Disallow: /select-style Disallow: /generate Disallow: /result Disallow: /signupconfirmation Disallow: /email-confirmed Disallow: /lite # UGC booking questionnaire — funnel step, not for indexing (wildcard matches /UGC-creator//book) Disallow: /UGC-creator/*/book # Same blocks for locale-prefixed variants Disallow: /en/auth Disallow: /ca/auth Disallow: /fr/auth Disallow: /tr/auth Disallow: /en/my-library Disallow: /ca/my-library Disallow: /fr/my-library Disallow: /tr/my-library Disallow: /en/settings Disallow: /ca/settings Disallow: /fr/settings Disallow: /tr/settings Disallow: /en/upload-dish Disallow: /ca/upload-dish Disallow: /fr/upload-dish Disallow: /tr/upload-dish Disallow: /en/select-style Disallow: /ca/select-style Disallow: /fr/select-style Disallow: /tr/select-style Disallow: /en/generate Disallow: /ca/generate Disallow: /fr/generate Disallow: /tr/generate Disallow: /en/result Disallow: /ca/result Disallow: /fr/result Disallow: /tr/result Disallow: /en/restaurant-images Disallow: /ca/restaurant-images Disallow: /fr/restaurant-images Disallow: /tr/restaurant-images Disallow: /en/signupconfirmation Disallow: /ca/signupconfirmation Disallow: /fr/signupconfirmation Disallow: /tr/signupconfirmation Disallow: /en/email-confirmed Disallow: /ca/email-confirmed Disallow: /fr/email-confirmed Disallow: /tr/email-confirmed Disallow: /en/lite Disallow: /ca/lite Disallow: /fr/lite Disallow: /tr/lite Disallow: /en/UGC-creator/*/book Disallow: /ca/UGC-creator/*/book Disallow: /fr/UGC-creator/*/book Disallow: /tr/UGC-creator/*/book Sitemap: https://www.foodyfocus.com/sitemap.xml