User-agent: * # WordPress / common attack surfaces (safe even if not WP) Disallow: /wp-admin/ Disallow: /wp-login.php Disallow: /xmlrpc.php # CodeIgniter 3 core directories (VERY IMPORTANT) Disallow: /application/ Disallow: /system/ Disallow: /vendor/ Disallow: /tests/ # CodeIgniter entry points & patterns Disallow: /index.php Disallow: /index.php/ Disallow: /*/index.php Disallow: /*/index.php/ # Common sensitive / noisy paths Disallow: /cgi-bin/ Disallow: /*.php$ Disallow: /?author= Disallow: /*?replytocom= # Bad bots / scrapers / spam crawlers User-agent: AhrefsBot Disallow: / User-agent: SemrushBot Disallow: / User-agent: MJ12bot Disallow: / User-agent: DotBot Disallow: / User-agent: BLEXBot Disallow: / User-agent: YandexBot Disallow: / User-agent: Baiduspider Disallow: / User-agent: Sogou Disallow: / User-agent: Exabot Disallow: / User-agent: SeznamBot Disallow: / User-agent: PetalBot Disallow: / # Explicitly allow major search engines User-agent: Googlebot Allow: / User-agent: Bingbot Allow: / User-agent: DuckDuckBot Allow: / User-agent: Applebot Allow: /