#
# Welcome to NVIDIA Omniverse Documentation Hub
# Last updated 5 Aug 2025 by LC

# Omniverse S3 Sitemap
Sitemap: https://docs.omniverse.nvidia.com/sitemap_index.xml

# AI / LLM crawlers & fetchers
User-agent: anthropic-ai
User-agent: Applebot
User-agent: AutoGPT
User-agent: Bytespider      # TikTok/Rudra LLM
User-agent: CCBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-web
User-agent: DuckDuckBot
User-agent: FacebookBot
User-agent: Google-Extended # Gemini / SGE
User-agent: GPTBot
User-agent: ia-archiver
User-agent: OAI-SearchBot
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Sogou
Allow: /*.md$
Allow: https://docs.omniverse.nvidia.com/llms.txt

User-agent: *
Disallow: /*?ncid*
Disallow: /*&ncid*
Disallow: /*?nvid*
Disallow: /*&nvid*
Disallow: /*?utm*
Disallow: /*&utm*