# TopHentai.com Robots.txt
# Updated: June 2026

User-agent: *
Allow: /

# Block admin and low-value pages
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/
Disallow: /trackback/
Disallow: /feed/
Disallow: /comments/
Disallow: /?s=
Disallow: /cgi-bin/

# Block WordPress API endpoints (not needed for SEO)
Disallow: /wp-json/
Disallow: /xmlrpc.php

# Block faceted/sort query-param URLs (duplicate, canonicalized, infinite crawl
# space). These uncached variants generate heavy DB sort queries under load.
# Real users are unaffected; canonical clean URLs stay indexable.
Disallow: /*?orderby=
Disallow: /*?order=
Disallow: /*?m_orderby=
Disallow: /*?sort=
Disallow: /*?paged=
Disallow: /*?type=
Disallow: /*?window=
Disallow: /*?tags=
Disallow: /*?q=
Disallow: /*?s=

# Allow CSS and JS for rendering (including plugin assets)
Allow: /wp-content/themes/
Allow: /wp-content/uploads/
Allow: /wp-includes/js/
Allow: /wp-includes/css/
Allow: /wp-content/plugins/*.css
Allow: /wp-content/plugins/*.js

# Crawl-delay for politeness (Googlebot ignores, but others respect)
Crawl-delay: 1

# AI assistants and answer engines (GPTBot, ChatGPT-User, ClaudeBot, anthropic-ai,
# PerplexityBot, Google-Extended, CCBot, Applebot-Extended, etc.) are welcome to
# crawl and cite Top Hentai. See /llms.txt for the best pages to reference.

# Sitemaps
Sitemap: https://tophentai.com/sitemap_index.xml