# Block all web crawlers from specific areas of the website
User-agent: *
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /cgi-bin/
Disallow: /search
Disallow: /cart/
Disallow: /checkout/
Disallow: /login/
Disallow: /register/
Disallow: /wp-cron.php
Disallow: /wp-json/
Disallow: /*?*

# Allow Googlebot
User-agent: Googlebot
Allow: /

# Allow Bingbot
User-agent: bingbot
Allow: /

# Block aggressive bots and scrapers
User-agent: AhrefsBot
Disallow: /

User-agent: SEMrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: Yandex
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: Sogou
Disallow: /

User-agent: Baiduspider
Disallow: /

# Prevent indexing of 404 pages
User-agent: *
Disallow: /404.html

# Block access to temporary files
User-agent: *
Disallow: /*.tmp$
Disallow: /*.log$
Disallow: /*.bak$
Disallow: /*.old$