# robots.txt for https://www.nextstepphysio.ca/
# Site type: WordPress Physiotherapy Clinic
# Notes: NextStep secondary domain
# Last updated: Sep 2026
# Maintained by LinkBuffer Studios — linkbufferstudios.com

User-agent: *
Allow: /
Allow: /llms.txt

# WordPress core — block admin and internals
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-includes/
Allow: /wp-includes/js/
Allow: /wp-includes/css/
Allow: /wp-includes/images/
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /xmlrpc.php
Disallow: /readme.html
Disallow: /license.txt

# Allow theme and plugin assets to render the page properly
Allow: /wp-content/uploads/
Allow: /wp-content/themes/*.css
Allow: /wp-content/themes/*.js
Allow: /wp-content/themes/*.png
Allow: /wp-content/themes/*.jpg
Allow: /wp-content/themes/*.jpeg
Allow: /wp-content/themes/*.webp
Allow: /wp-content/themes/*.svg
Allow: /wp-content/plugins/*.css
Allow: /wp-content/plugins/*.js
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/
Disallow: /wp-content/upgrade/

# Block low-value WordPress paths
Disallow: /trackback/
Disallow: /*/trackback/
Disallow: /tag/
Disallow: /author/
Disallow: /?author=
Disallow: /comments/feed/

# Block internal search and tracking parameters
Disallow: /?s=
Disallow: /*?s=
Disallow: /*?*replytocom=
Disallow: /*?*share=
Disallow: /*?*print=
Disallow: /*?*utm_
Disallow: /*?*fbclid=
Disallow: /*?*gclid=
Disallow: /*?*msclkid=

# ============================================
# AI / LLM CRAWLERS — Explicitly allowed
# Listed alphabetically by provider
# ============================================

# Amazon — Alexa / Amazon AI
User-agent: Amazonbot
Allow: /

# Anthropic — Claude (3 active bots as of Feb 2026)
# ClaudeBot = training data collection
# Claude-User = real-time user queries
# Claude-SearchBot = Claude search indexing
User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

# Apple — Siri / Apple Intelligence
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# ByteDance — TikTok search
User-agent: Bytespider
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

# Google — AI Overview / Gemini training opt-in
User-agent: Google-Extended
Allow: /

User-agent: GoogleOther
Allow: /

# Meta — Meta AI
User-agent: Meta-ExternalAgent
Allow: /

User-agent: FacebookBot
Allow: /

# OpenAI — ChatGPT (3 active bots)
# GPTBot = training
# ChatGPT-User = user-triggered fetches
# OAI-SearchBot = ChatGPT search
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: SemrushBot
Allow: /

User-agent: SemrushBot-SA
Allow: /

# ============================================
# AGGRESSIVE / NON-AI SCRAPERS — Blocked
# These hammer the server with no return value
# ============================================
User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: SeznamBot
Disallow: /

# ============================================
# SITEMAP & RESOURCES
# ============================================

# XML sitemap (for search engines)
Sitemap: https://www.nextstepphysio.ca/sitemap_index.xml

# LLM-readable site summary (deployed at root)
# Reference: https://www.nextstepphysio.ca/llms.txt