# See https://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
User-agent: *
Allow: /
# No trailing slash, deliberately: "/admin-giocatoli/" does NOT match the bare
# "/admin-giocatoli", which 302s to the sign-in page. Redirects are 20% of our
# crawl budget per GSC crawl stats, and this was one of the doors left open.
Disallow: /admin-giocatoli
Disallow: /api/
Disallow: /rails/
Disallow: /users/
# Amazon affiliate cloaked redirects — must NOT be crawled (they 302 to amazon.com)
Disallow: /go/
Disallow: /affiliate/
# Curated playlist share links — JS soft-redirects to /, Google flags as "Page with redirect"
Disallow: /*?playlist=
# Mobile share target (get "share" => pages#home). Every /share?url=... is a
# parameterised duplicate of the homepage; GSC crawled one on 2 Aug 2026 and
# filed it under "Crawled - currently not indexed". Nothing to index here.
Disallow: /share
# Background-audio diagnostic harness (bg-audio-test.html + its tone/manifest/video
# assets). Dev tooling that lives in public/, so it is publicly reachable — keep it
# out of the index: an indexed test page dilutes the site's topical relevance and
# would surface as a broken-looking result. The page also carries .
Disallow: /bg-audio-
Disallow: /bg-nosleep.mp4
# Explicit disallow for /api/transcripts/ — Lighthouse + Googlebot trigger
# 1.25MB heap spikes on Render datacenter IPs (now also UA-filtered server-side).
User-agent: Googlebot
Disallow: /api/transcripts/
User-agent: Bingbot
Disallow: /api/transcripts/
User-agent: Chrome-Lighthouse
Disallow: /api/transcripts/
User-agent: AhrefsBot
Disallow: /api/transcripts/
User-agent: SemrushBot
Disallow: /api/transcripts/
# Social preview bots — allow share previews even for blocked playlist URLs
User-agent: facebookexternalhit
Allow: /
User-agent: Twitterbot
Allow: /
User-agent: LinkedInBot
Allow: /
User-agent: WhatsApp
Allow: /
User-agent: Discordbot
Allow: /
User-agent: Slackbot
Allow: /
User-agent: TelegramBot
Allow: /
Sitemap: https://mixtuby.com/sitemap.xml