# robots.txt for www.tanu.be
# Advanced configuration for search engines and AI crawlers

# -------------------------------------------------------
# GLOBAL SETTINGS
# -------------------------------------------------------
User-agent: *
Allow: /

# Prevent crawling of internal system files or sensitive paths (examples—adjust as needed)
Disallow: /wp-admin/
Disallow: /cgi-bin/
Disallow: /tmp/
Disallow: /private/
Disallow: /drafts/

# Allow AI agents and search engines to fetch metadata
Allow: /llms.txt

# Optional: Control crawl frequency
Crawl-delay: 2


# -------------------------------------------------------
# MAJOR SEARCH ENGINE DIRECTIVES
# -------------------------------------------------------

# Google
User-agent: Googlebot
Allow: /

# Googlebot Images
User-agent: Googlebot-Image
Allow: /

# Google-Extended (used for AI training datasets)
# Set to Allow or Disallow based on your preference
User-agent: Google-Extended
Disallow: /

# Bing
User-agent: Bingbot
Allow: /

# Bing AI / Microsoft crawlers
User-agent: BingPreview
Allow: /


# -------------------------------------------------------
# AI / LLM CRAWLERS
# -------------------------------------------------------

# OpenAI GPTBot (ChatGPT data crawling bot)
# https://platform.openai.com/docs/gptbot
User-agent: GPTBot
Disallow: /

# Anthropic Claude CCBot
User-agent: CCBot
Disallow: /

# Perplexity AI crawler
User-agent: PerplexityBot
Disallow: /

# Amazon Q / AGI bot (formerly Amazonbot)
User-agent: Amazonbot
Disallow: /


# -------------------------------------------------------
# SOCIAL MEDIA CRAWLERS
# -------------------------------------------------------

# Facebook
User-agent: Facebot
Allow: /

# LinkedIn
User-agent: LinkedInBot
Allow: /

# Twitter / X
User-agent: Twitterbot
Allow: /


# -------------------------------------------------------
# SITEMAP LOCATIONS
# -------------------------------------------------------
# Add your sitemap file(s) if available
# Sitemap: https://www.tanu.be/sitemap.xml


# -------------------------------------------------------
# LEGAL + TRANSPARENCY
# -------------------------------------------------------
# This robots.txt is provided to clarify which bots may access content on www.tanu.be
# For AI crawlers, llms.txt supplies additional metadata:
# https://www.tanu.be/llms.txt