# HandsOnTech — https://www.handsontech.io
#
# Public commercial pages are open to search engines and AI crawlers.
# Admin and API are excluded.
#
# Proof content is private by policy:
#   /portfolio/          case study and portfolio pages
#   /portfolio-category  legacy entry point (301s into /portfolio/)
#   /case-studies/       static proof assets (images, video, Figma exports)
# These paths are noindex, carry an X-Robots-Tag header, are absent from the XML
# sitemap and are not listed in llms.txt. Patterns are kept narrow so no
# commercial page is affected. Source of truth: src/lib/proof-routes.ts.
#
# ---------------------------------------------------------------------------
# TEMPORARY, 2026-08-20: DE-INDEXING WINDOW FOR /portfolio/
#
# Googlebot, Googlebot-Image and Bingbot are deliberately ALLOWED to crawl
# /portfolio/ right now. A noindex directive only takes effect if the crawler is
# permitted to fetch the page and read it; blocking the path in robots.txt would
# freeze any already-indexed URLs in place instead of removing them.
#
# AI crawlers stay blocked. They hold no search index to purge, so opening the
# path to them would deliver private client work into training and retrieval
# corpora for no de-indexing benefit whatsoever.
#
# REVERT WHEN: Search Console reports 0 indexed URLs under /portfolio/.
# HOW: restore `Disallow: /portfolio/`, `Disallow: /portfolio-category` and
# `Disallow: /case-studies/` to the Googlebot, Googlebot-Image and Bingbot
# groups below. Nothing else needs to change.
# ---------------------------------------------------------------------------

User-agent: *
Allow: /
Allow: /services/
Allow: /hire/
Allow: /tools/
Allow: /locations/
Allow: /industries/
Allow: /blog/
Disallow: /admin/
Disallow: /api/
Disallow: /portfolio/
Disallow: /portfolio-category
Disallow: /case-studies/

# ---------- Search engine crawlers ----------

# TEMPORARY de-indexing window: must crawl /portfolio/ to see the noindex and drop the URLs
User-agent: Googlebot
Allow: /
Allow: /portfolio/
Allow: /case-studies/
Disallow: /admin/
Disallow: /api/

# TEMPORARY de-indexing window: must crawl to drop already-indexed proof imagery
User-agent: Googlebot-Image
Allow: /
Allow: /portfolio/
Allow: /case-studies/
Disallow: /admin/
Disallow: /api/

# TEMPORARY de-indexing window: same de-indexing mechanic as Googlebot
User-agent: Bingbot
Allow: /
Allow: /portfolio/
Allow: /case-studies/
Disallow: /admin/
Disallow: /api/

# ---------- AI training and AI answer crawlers ----------

User-agent: GPTBot
Allow: /
Allow: /services/
Allow: /hire/
Allow: /tools/
Allow: /locations/
Allow: /industries/
Allow: /blog/
Disallow: /admin/
Disallow: /api/
Disallow: /portfolio/
Disallow: /portfolio-category
Disallow: /case-studies/

User-agent: OAI-SearchBot
Allow: /
Allow: /services/
Allow: /hire/
Allow: /tools/
Allow: /locations/
Allow: /industries/
Allow: /blog/
Disallow: /admin/
Disallow: /api/
Disallow: /portfolio/
Disallow: /portfolio-category
Disallow: /case-studies/

User-agent: ChatGPT-User
Allow: /
Allow: /services/
Allow: /hire/
Allow: /tools/
Allow: /locations/
Allow: /industries/
Allow: /blog/
Disallow: /admin/
Disallow: /api/
Disallow: /portfolio/
Disallow: /portfolio-category
Disallow: /case-studies/

User-agent: ClaudeBot
Allow: /
Allow: /services/
Allow: /hire/
Allow: /tools/
Allow: /locations/
Allow: /industries/
Allow: /blog/
Disallow: /admin/
Disallow: /api/
Disallow: /portfolio/
Disallow: /portfolio-category
Disallow: /case-studies/

User-agent: Claude-User
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /portfolio/
Disallow: /portfolio-category
Disallow: /case-studies/

User-agent: Claude-SearchBot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /portfolio/
Disallow: /portfolio-category
Disallow: /case-studies/

User-agent: PerplexityBot
Allow: /
Allow: /services/
Allow: /hire/
Allow: /tools/
Allow: /locations/
Allow: /industries/
Allow: /blog/
Disallow: /admin/
Disallow: /api/
Disallow: /portfolio/
Disallow: /portfolio-category
Disallow: /case-studies/

User-agent: Perplexity-User
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /portfolio/
Disallow: /portfolio-category
Disallow: /case-studies/

User-agent: Google-Extended
Allow: /
Allow: /services/
Allow: /hire/
Allow: /tools/
Allow: /locations/
Allow: /industries/
Allow: /blog/
Disallow: /admin/
Disallow: /api/
Disallow: /portfolio/
Disallow: /portfolio-category
Disallow: /case-studies/

User-agent: CCBot
Allow: /
Allow: /services/
Allow: /hire/
Allow: /tools/
Allow: /locations/
Allow: /industries/
Allow: /blog/
Disallow: /admin/
Disallow: /api/
Disallow: /portfolio/
Disallow: /portfolio-category
Disallow: /case-studies/

User-agent: Amazonbot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /portfolio/
Disallow: /portfolio-category
Disallow: /case-studies/

User-agent: Applebot-Extended
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /portfolio/
Disallow: /portfolio-category
Disallow: /case-studies/

User-agent: meta-externalagent
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /portfolio/
Disallow: /portfolio-category
Disallow: /case-studies/

User-agent: Bytespider
Allow: /
Allow: /services/
Allow: /hire/
Allow: /tools/
Allow: /locations/
Allow: /industries/
Allow: /blog/
Disallow: /admin/
Disallow: /api/
Disallow: /portfolio/
Disallow: /portfolio-category
Disallow: /case-studies/

Sitemap: https://www.handsontech.io/sitemap-index.xml