# Robots.txt for gtmengine.ai # Updated for modern search engines and AI crawlers # Content Signals - Declare AI content usage preferences # See: https://contentsignals.org/ Content-Signal: ai-train=yes, search=yes, ai-input=yes # Allow all well-behaved crawlers by default User-agent: * Allow: / # Sitemap location Sitemap: https://www.gtmengine.ai/sitemap.xml # LLMs index (https://llmstxt.org/) # Llms-txt: https://www.gtmengine.ai/llms.txt # Llms-full-txt: https://www.gtmengine.ai/llms-full.txt # Block common spam/malicious bots User-agent: AhrefsBot Disallow: / User-agent: MJ12bot Disallow: / User-agent: DotBot Disallow: / User-agent: SemrushBot Disallow: / User-agent: MegaIndex Disallow: / # Block admin and private areas (if any exist) User-agent: * Disallow: /admin/ Disallow: /studio/ Disallow: /_next/ Disallow: /api/ Disallow: /investors Disallow: /data-room # Allow RSS feeds Allow: /rss/ # Allow AI crawlers for training (OpenAI, Anthropic, Google, etc.) # Note: Individual pages can override this with noindex meta tags User-agent: GPTBot Allow: / User-agent: Google-Extended Allow: / User-agent: CCBot Allow: / User-agent: anthropic-ai Allow: / User-agent: Claude-Web Allow: / User-agent: PerplexityBot Allow: / User-agent: cohere-ai Allow: / User-agent: FacebookBot Allow: / User-agent: Meta-ExternalAgent Allow: / User-agent: Applebot-Extended Allow: / User-agent: Diffbot Allow: / User-agent: YouBot Allow: / User-agent: Bytespider Allow: / User-agent: ImagesiftBot Allow: / User-agent: OAI-SearchBot Allow: / # Crawl delay for aggressive crawlers User-agent: * Crawl-delay: 1