# ===========================================
# robots.txt for https://digitaltekblog.com
# Optimized for WordPress + Rank Math + GeneratePress
# Last Updated: November 2025
# ===========================================

User-agent: *
# --- Allow essential assets ---
Allow: /wp-content/uploads/
Allow: /wp-admin/admin-ajax.php

# --- Block sensitive or internal system directories ---
Disallow: /wp-admin/
Disallow: /cgi-bin/
Disallow: /private/
Disallow: /temp/
Disallow: /unused-directory/

# --- Block low-value or duplicate pages ---
Disallow: /?s=
Disallow: /search/
Disallow: /*/feed/
Disallow: /*/trackback/
Disallow: /*/amp/
Disallow: /page/

# --- Allow SEO-beneficial sections ---
Allow: /category/
Allow: /tag/

# --- Sitemap for search engines ---
Sitemap: https://digitaltekblog.com/sitemap_index.xml

# ===========================================
# Notes:
# 1. Fake or spam URLs are now hard-blocked via .htaccess (403/410).
# 2. This robots.txt keeps crawl budget focused on valid posts and pages.
# 3. Test in Google Search Console → Robots.txt Tester.
# ===========================================