# Content Signals (https://contentsignals.org/). These restate the policy
# below in a machine-readable form: search and live AI answers are allowed,
# AI training is not. Crawlers that do not know this line ignore it
# (RFC 9309 section 2.2.4).
#
# As a condition of accessing this website, you agree to abide by the
# following content signals:
#
# (a)  If a content-signal = yes, you may collect content for the
#      corresponding use.
# (b)  If a content-signal = no, you may not collect content for the
#      corresponding use.
# (c)  If the website operator does not include a content signal for a
#      corresponding use, the website operator neither grants nor restricts
#      permission via content signal with respect to the corresponding use.
#
# The content signals and their meanings are:
#
# search: building a search index and providing search results (e.g., returning
#         hyperlinks and short excerpts from your website's contents). Search
#         does not include providing AI-generated search summaries.
# ai-input: inputting content into one or more AI models (e.g., retrieval
#           augmented generation, grounding, or other real-time taking of
#           content for generative AI search answers).
# ai-train: training or fine-tuning AI models.
#
# ANY RESTRICTIONS EXPRESSED VIA CONTENT SIGNALS ARE EXPRESS RESERVATIONS OF
# RIGHTS UNDER ARTICLE 4 OF THE EUROPEAN UNION DIRECTIVE 2019/790 ON COPYRIGHT
# AND RELATED RIGHTS IN THE DIGITAL SINGLE MARKET.

# AI training opt-out. SimTool.com content may not be used to train AI
# models. These agents (or control tokens) exist only for model training or
# AI dataset building; the search and answer crawlers below are unaffected.
# Google-Extended and Applebot-Extended are control tokens, not crawlers:
# Googlebot and Applebot still crawl for Search. anthropic-ai is a legacy
# Anthropic token. These groups come first so that simple prefix-matching
# parsers cannot confuse them with the allowed agents below.
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: CCBot
User-agent: meta-externalagent
User-agent: Amazonbot
User-agent: AI2Bot
User-agent: cohere-training-data-crawler
User-agent: Webzio-Extended
Disallow: /

# Bytespider is reported to ignore multi-agent groups, so it gets its own.
User-agent: Bytespider
Disallow: /

# Search and AI answer crawlers we want, named explicitly. A named group
# replaces the * group for that crawler, so it repeats the same rules.
User-agent: Googlebot
User-agent: Bingbot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: Applebot
User-agent: Amzn-SearchBot
User-agent: Meta-WebIndexer
# A named group does not inherit the * group, so it repeats the signal.
Content-Signal: search=yes, ai-input=yes, ai-train=no
# Everything not disallowed below is allowed (no blanket "Allow: /", which
# first-match parsers would apply before the Disallows).
# Explicit allows for primary content sections
Allow: /article/
Allow: /categories/
Allow: /tools/
Allow: /latest-news/
Allow: /about/
Allow: /contact/
Allow: /privacy-policy/
# Allow sitemap, feed, and LLM guidance file
Allow: /sitemap.xml
Allow: /API/sitemap.php
Allow: /feed/
Allow: /llms.txt
# Internal search results should not be crawled (crawl-budget waste;
# pages also carry noindex)
Disallow: /search/
# API and admin areas are not content. Directory-level rules only;
# do not list individual endpoint filenames here (this file is public).
# The more specific Allow for /API/sitemap.php above still applies.
Disallow: /API/
Disallow: /admin/

# All other crawlers
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=no
# Everything not disallowed below is allowed (no blanket "Allow: /", which
# first-match parsers would apply before the Disallows).
# Explicit allows for primary content sections
Allow: /article/
Allow: /categories/
Allow: /tools/
Allow: /latest-news/
Allow: /about/
Allow: /contact/
Allow: /privacy-policy/
# Allow sitemap, feed, and LLM guidance file
Allow: /sitemap.xml
Allow: /API/sitemap.php
Allow: /feed/
Allow: /llms.txt
# Internal search results should not be crawled (crawl-budget waste;
# pages also carry noindex)
Disallow: /search/
# API and admin areas are not content. Directory-level rules only;
# do not list individual endpoint filenames here (this file is public).
# The more specific Allow for /API/sitemap.php above still applies.
Disallow: /API/
Disallow: /admin/

Sitemap: https://www.simtool.com/sitemap.xml