# =========================================
# LangProtect Robots.txt
# https://www.langprotect.com/
# =========================================
 
 
# -----------------------------------------
# Default Crawlers
# -----------------------------------------
 
User-agent: *
Allow: /
 
Disallow: /thank-you
Disallow: /blog/author/None
Disallow: /solutions/ai-consulting-for-legal
Disallow: /solutions/ai-consulting-for-technology
Disallow: /ai-consulting-for-technology
Disallow: /blog/try-guardia.langprotect.com
Disallow: /blog/langprotect.com/contact-us
Disallow: /html/body/main/article/h2
Disallow: /html/body/div[1]/main/article/h2
Disallow: /html/body//h2
Disallow: /html/body//p
Disallow: /html/head/title
Disallow: /html/body//h1
Disallow: /get-started
Disallow: /healthcare
 
 
# -----------------------------------------
# AI Search & Retrieval Crawlers
# -----------------------------------------
 
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
 
Allow: /
 
Disallow: /thank-you
Disallow: /blog/author/None
Disallow: /solutions/ai-consulting-for-legal
Disallow: /solutions/ai-consulting-for-technology
Disallow: /ai-consulting-for-technology
Disallow: /blog/try-guardia.langprotect.com
Disallow: /blog/langprotect.com/contact-us
Disallow: /html/body/main/article/h2
Disallow: /html/body/div[1]/main/article/h2
Disallow: /html/body//h2
Disallow: /html/body//p
Disallow: /html/head/title
Disallow: /html/body//h1
Disallow: /get-started
Disallow: /healthcare
 
 
# -----------------------------------------
# AI Training / Model Improvement Crawlers
# -----------------------------------------
 
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Google-Extended
 
Allow: /
 
Disallow: /thank-you
Disallow: /blog/author/None
Disallow: /solutions/ai-consulting-for-legal
Disallow: /solutions/ai-consulting-for-technology
Disallow: /ai-consulting-for-technology
Disallow: /blog/try-guardia.langprotect.com
Disallow: /blog/langprotect.com/contact-us
Disallow: /html/body/main/article/h2
Disallow: /html/body/div[1]/main/article/h2
Disallow: /html/body//h2
Disallow: /html/body//p
Disallow: /html/head/title
Disallow: /html/body//h1
Disallow: /get-started
Disallow: /healthcare
 
 
# -----------------------------------------
# Bulk Dataset Crawlers
# -----------------------------------------
 
User-agent: CCBot
Disallow: /
 
 
# -----------------------------------------
# Sitemap
# -----------------------------------------
 
Sitemap: https://www.langprotect.com/sitemap.xml