# theyetilife.com robots.txt — v5 (2026-04-25) # AI-friendly: all major LLM crawlers explicitly allowed. # Tag pages + paginated archive pages now CRAWLABLE so noindex meta is discoverable. User-agent: * Allow: /products/*?variant= Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /orders Disallow: /checkouts/ Disallow: /checkout Disallow: /carts Disallow: /account Disallow: /collections/*sort_by* Disallow: /collections/*filter* Allow: /collections/ Disallow: /search Disallow: /?*oseid= Disallow: /preview_theme_id Disallow: /preview_script_id # Transactional / utility pages — should not be indexed Disallow: /pages/thank-you Disallow: /pages/order-confirmation Disallow: /apple-app-site-association Disallow: /.well-known/shopify/monorail Disallow: /cdn/wpm/*.js # Block only problematic dynamic product URLs Disallow: /products/*-*-remote Sitemap: https://theyetilife.com/sitemap.xml User-agent: AdsBot-Google Allow: /products/*?variant= Disallow: /checkouts/ Disallow: /checkout Disallow: /cart Disallow: /orders # === AI / LLM CRAWLERS — full content access === # These bots power ChatGPT, Claude, Perplexity, Gemini AI mode, etc. # Only utility/transactional pages blocked. User-agent: GPTBot Allow: / Disallow: /cart Disallow: /account Disallow: /checkouts/ Disallow: /checkout Disallow: /orders User-agent: ChatGPT-User Allow: / Disallow: /cart Disallow: /account Disallow: /checkouts/ Disallow: /checkout Disallow: /orders User-agent: OAI-SearchBot Allow: / Disallow: /cart Disallow: /account Disallow: /checkouts/ Disallow: /checkout Disallow: /orders User-agent: Google-Extended Allow: / Disallow: /cart Disallow: /account Disallow: /checkouts/ Disallow: /checkout Disallow: /orders User-agent: PerplexityBot Allow: / Disallow: /cart Disallow: /account Disallow: /checkouts/ Disallow: /checkout Disallow: /orders User-agent: Perplexity-User Allow: / Disallow: /cart Disallow: /account Disallow: /checkouts/ Disallow: /checkout Disallow: /orders User-agent: anthropic-ai Allow: / Disallow: /cart Disallow: /account Disallow: /checkouts/ Disallow: /checkout Disallow: /orders User-agent: ClaudeBot Allow: / Disallow: /cart Disallow: /account Disallow: /checkouts/ Disallow: /checkout Disallow: /orders User-agent: Claude-Web Allow: / Disallow: /cart Disallow: /account Disallow: /checkouts/ Disallow: /checkout Disallow: /orders User-agent: CCBot Allow: / Disallow: /cart Disallow: /account Disallow: /checkouts/ Disallow: /checkout Disallow: /orders User-agent: Applebot-Extended Allow: / Disallow: /cart Disallow: /account Disallow: /checkouts/ Disallow: /checkout Disallow: /orders User-agent: Bytespider Allow: / Disallow: /cart Disallow: /account Disallow: /checkouts/ Disallow: /checkout Disallow: /orders User-agent: Meta-ExternalAgent Allow: / Disallow: /cart Disallow: /account Disallow: /checkouts/ Disallow: /checkout Disallow: /orders User-agent: Meta-ExternalFetcher Allow: / Disallow: /cart Disallow: /account Disallow: /checkouts/ Disallow: /checkout Disallow: /orders User-agent: YouBot Allow: / Disallow: /cart Disallow: /account Disallow: /checkouts/ Disallow: /checkout Disallow: /orders User-agent: cohere-ai Allow: / Disallow: /cart Disallow: /account Disallow: /checkouts/ Disallow: /checkout Disallow: /orders User-agent: Diffbot Allow: / Disallow: /cart Disallow: /account Disallow: /checkouts/ Disallow: /checkout Disallow: /orders User-agent: DuckAssistBot Allow: / Disallow: /cart Disallow: /account Disallow: /checkouts/ Disallow: /checkout Disallow: /orders