# robots.txt for https://indiasocialbook.com

# -------------------------------------------------------------
# 1. BLOCK AGGRESSIVE THIRD-PARTY SEO SCRAPERS (Saves VPS CPU/RAM)
# -------------------------------------------------------------
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: Semrush
User-agent: MJ12bot
User-agent: DotBot
User-agent: PetalBot
User-agent: Bytespider
User-agent: ZoominfoBot
Disallow: /

# -------------------------------------------------------------
# 2. GOOGLE IMAGE CRAWLER (Kept as per your original configuration)
# -------------------------------------------------------------
User-agent: Googlebot-Image
Disallow: /

# -------------------------------------------------------------
# 3. SEARCH ENGINES & CRAWL BUDGET OPTIMIZATION (Googlebot, Bingbot, etc.)
# -------------------------------------------------------------
User-agent: *

# XenForo System & Security Endpoints (Fixed admin.php syntax)
Disallow: /cgi-bin/
Disallow: /admin.php
Disallow: /install/
Disallow: /login/
Disallow: /register/
Disallow: /lost-password/
Disallow: /account/

# Cut 95.5% Crawl Waste (Crawl loops, redirects, infinite pagination)
Disallow: /redirect*
Disallow: /whats-new/
Disallow: /find-new/
Disallow: /feeds/
Disallow: /css.php
Disallow: /search/

# Prevent Duplicate Thread Anchors & Post Links
Disallow: /goto/
Disallow: /posts/
Disallow: /members/
Disallow: /attachments/

# Explicitly Allow Content Directories
Allow: /threads/
Allow: /videos/
Allow: /data/

# Block Leaked Add-on Management Endpoints
Disallow: /*/audfeeds-manage
Disallow: *audfeeds-manage

# Block Internal Search Engines & Profile Posts
Disallow: /search/
Disallow: /profile-posts/

# -------------------------------------------------------------
# 4. SITEMAP DECLARATIONS
# -------------------------------------------------------------
Sitemap: https://indiasocialbook.com/sitemap.php
Sitemap: https://indiasocialbook.com/seotools/sitemap-generator/indiasocialbook.com_sitemap.xml
Sitemap: https://indiasocialbook.com/sitemap-downdetector.php