User-agent: ia_archiver Disallow: / User-agent: archive.org_bot Disallow: / User-agent: archive.is Disallow: / User-agent: archive.ph Disallow: / User-agent: archive.today Disallow: / # Allow major search engines User-agent: Googlebot Allow: / User-agent: Bingbot Allow: / User-agent: Slurp Allow: / User-agent: YandexBot Allow: / User-agent: Baiduspider Allow: / # Optional: allow SEO crawlers (if you want them) User-agent: AhrefsBot Allow: / User-agent: MJ12bot Allow: / User-agent: Ezooms Allow: / User-agent: aiHitBot Allow: /