# robots.txt for acscricket.com
# Goal: Allow search engine indexing; block AI training crawlers;
#       protect WordPress admin areas.

# ---------------------------------------------------------------
# Standard search engines — full access
# ---------------------------------------------------------------
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

# ---------------------------------------------------------------
# WordPress admin/sensitive paths — block all crawlers
# ---------------------------------------------------------------
User-agent: *
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-json/
Disallow: /xmlrpc.php
Disallow: /?author=
Disallow: /trackback/
Disallow: /feed/
Disallow: /comments/feed/

# ---------------------------------------------------------------
# AI TRAINING crawlers — blocked
# These bots collect content specifically for LLM training.
# Blocking them does not affect search engine rankings.
# ---------------------------------------------------------------

# OpenAI — training crawler
User-agent: GPTBot
Disallow: /

# Anthropic — training crawlers
User-agent: ClaudeBot
Disallow: /
User-agent: anthropic-ai
Disallow: /

# Google — Gemini/AI training crawler (separate from Googlebot)
User-agent: Google-Extended
Disallow: /

# Common Crawl — open dataset used by many AI labs for training
User-agent: CCBot
Disallow: /

# Meta AI
User-agent: FacebookBot
Disallow: /

# Apple AI
User-agent: Applebot-Extended
Disallow: /

# Cohere
User-agent: cohere-ai
Disallow: /

# AI2 (Allen Institute)
User-agent: AI2Bot
Disallow: /

# Diffbot
User-agent: Diffbot
Disallow: /

# ByteDance / Doubao (known for ignoring robots.txt — block here as first layer;
# consider WAF/server-level blocking as well)
User-agent: Bytespider
Disallow: /

# ---------------------------------------------------------------
# AI SEARCH / RETRIEVAL crawlers — allowed
# These bots power AI-assisted search features (not training).
# Allowing them keeps the site visible in AI-assisted search results.
# ---------------------------------------------------------------

# OpenAI search indexing (not training)
User-agent: OAI-SearchBot
Allow: /

# Perplexity search
User-agent: PerplexityBot
Allow: /

# ---------------------------------------------------------------
# Sitemap
# ---------------------------------------------------------------
Sitemap: https://acscricket.com/sitemap.xml

:User-Agent: *
Disallow: /Archive/Scorecards/hidden/
Disallow: /Publications_Progress/
Disallow: /Experimental/
Disallow: /famous_cricketers/
Disallow: /journals/1/files
Disallow: /journals/2/files
Disallow: /journals/3/files
Disallow: /journals/4/files
Disallow: /journals/5/files
Disallow: /journals/6/files
Disallow: /journals/7/files
Disallow: /journals/8/files
Disallow: /journals/9/files
Disallow: /journals/10/files
Disallow: /journals/11/files
Disallow: /journals/12/files
Disallow: /journals/13/files
Disallow: /journals/14/files
Disallow: /journals/15/files
Disallow: /journals/16/files
Disallow: /journals/17/files
Disallow: /journals/18/files
Disallow: /journals/19/files
Disallow: /journals/20/files
Disallow: /journals/21/files
Disallow: /journals/22/files
Disallow: /journals/23/files
Disallow: /journals/24/files
Disallow: /journals/25/files
Disallow: /journals/26/files
Disallow: /journals/27/files
Disallow: /journals/28/files
Disallow: /journals/29/files
Disallow: /journals/30/files
Disallow: /journals/31/files
Disallow: /journals/32/files
Disallow: /journals/33/files
Disallow: /journals/34/files
Disallow: /journals/35/files
Disallow: /journals/36/files
Disallow: /journals/37/files
Disallow: /journals/38/files
Disallow: /journals/39/files
Disallow: /journals/40/files
Disallow: /journals/41/files
Disallow: /journals/42/files
Disallow: /journals/43/files
Disallow: /journals/44/files
Disallow: /journals/45/files
Disallow: /journals/46/files
Disallow: /journals/47/files
Disallow: /journals/48/files
Disallow: /journals/49/files
Disallow: /journals/50/files
Disallow: /journals/51/files
Disallow: /journals/52/files
Disallow: /journals/53/files
Disallow: /journals/54/files
Disallow: /journals/55/files
Disallow: /journals/56/files
Disallow: /journals/57/files
Disallow: /journals/58/files
Disallow: /journals/59/files
Disallow: /journals/60/files
Disallow: /journals/61/files
Disallow: /journals/62/files
Disallow: /journals/63/files
Disallow: /journals/64/files
Disallow: /journals/65/files
Disallow: /journals/66/files
Disallow: /journals/67/files
Disallow: /journals/68/files
Disallow: /journals/69/files
Disallow: /journals/70/files
Disallow: /journals/71/files
Disallow: /journals/72/files
Disallow: /journals/73/files
Disallow: /journals/74/files
Disallow: /journals/75/files
Disallow: /journals/76/files
Disallow: /journals/77/files
Disallow: /journals/78/files
Disallow: /journals/79/files
Disallow: /journals/80/files
Disallow: /journals/81/files
Disallow: /journals/82/files
Disallow: /journals/83/files
Disallow: /journals/84/files
Disallow: /journals/85/files
Disallow: /journals/86/files
Disallow: /journals/87/files
Disallow: /journals/88/files
Disallow: /journals/89/files
Disallow: /journals/90/files
Disallow: /journals/91/files
Disallow: /journals/92/files
Disallow: /journals/93/files
Disallow: /journals/94/files
Disallow: /journals/95/files
Disallow: /journals/96/files
Disallow: /journals/97/files
Disallow: /journals/98/files
Disallow: /journals/99/files
Disallow: /journals/100/files
Disallow: /journals/101/files
Disallow: /journals/102/files
Disallow: /journals/103/files
Disallow: /journals/104/files
Disallow: /journals/105/files
Disallow: /journals/106/files
Disallow: /journals/107/files
Disallow: /journals/108/files
Disallow: /journals/109/files
Disallow: /journals/110/files
Disallow: /journals/111/files
Disallow: /journals/112/files
Disallow: /journals/113/files
Disallow: /journals/114/files
Disallow: /journals/115/files
Disallow: /journals/116/files
Disallow: /journals/117/files
Disallow: /journals/118/files
Disallow: /journals/119/files
Disallow: /journals/120/files
Disallow: /journals/121/files
Disallow: /journals/122/files
Disallow: /journals/123/files
Disallow: /journals/124/files
Disallow: /journals/125/files
Disallow: /journals/126/files
Disallow: /journals/127/files
Disallow: /journals/128/files
Disallow: /journals/129/files
Disallow: /journals/130/files
Disallow: /journals/131/files
Disallow: /journals/132/files
Disallow: /journals/133/files
Disallow: /journals/134/files
Disallow: /journals/135/files
Disallow: /journals/136/files
Disallow: /journals/137/files
Disallow: /journals/138/files
Disallow: /journals/139/files
Disallow: /journals/140/files
Disallow: /journals/141/files
Disallow: /journals/142/files
Disallow: /journals/143/files
Disallow: /journals/144/files
Disallow: /journals/145/files
Disallow: /journals/146/files
Disallow: /journals/147/files
Disallow: /journals/148/files
Disallow: /journals/149/files
Disallow: /journals/150/files
Disallow: /journals/151/files
Disallow: /journals/152/files
Disallow: /journals/153/files
Disallow: /journals/154/files
Disallow: /journals/155/files
Disallow: /journals/index_1-36/files
Disallow: /journals/index_1-77/files
Disallow: /journals/index_1-100/files
User-Agent: *
Crawl-Delay: 10
User-Agent: *
Crawl-Delay: 10