# -----------------------------------------------
# Social Media Crawlers
# -----------------------------------------------

User-agent: Twitterbot
Disallow:

User-agent: facebookexternalhit
Disallow:

User-agent: LinkedInBot
Disallow:

User-agent: Pinterestbot
Disallow:

User-agent: WhatsApp
Disallow:

User-agent: Slackbot
Disallow:

User-agent: Discordbot
Disallow:

User-agent: TelegramBot
Disallow:

User-agent: Snapchat
Disallow:

User-agent: Mastodon
Disallow:

User-agent: Applebot-Extended
Disallow:

# -----------------------------------------------
# Search Engines
# -----------------------------------------------

User-agent: Googlebot
Disallow:

User-agent: Bingbot
Disallow:

User-agent: DuckDuckBot
Disallow:

User-agent: YandexBot
Disallow:

User-agent: Baiduspider
Disallow:

User-agent: Applebot
Disallow:

# -----------------------------------------------
# AI / LLM Crawlers
# -----------------------------------------------

User-agent: GPTBot
Disallow:

User-agent: ChatGPT-User
Disallow:

User-agent: ClaudeBot
Disallow:

User-agent: anthropic-ai
Disallow:

User-agent: Google-Extended
Disallow:

User-agent: CCBot
Disallow:

User-agent: PerplexityBot
Disallow:

User-agent: YouBot
Disallow:

User-agent: Omgilibot
Disallow:

User-agent: Meta-ExternalAgent
Disallow:

User-agent: Diffbot
Disallow:

User-agent: *
Disallow: /