# robots.txt for media.kinkykink.com (Cloudflare R2)

# Allow all major search engine crawlers
User-agent: Googlebot
Disallow:

User-agent: Bingbot
Disallow:

User-agent: DuckDuckBot
Disallow:

User-agent: Applebot
Disallow:

User-agent: CCBot
Disallow:

User-agent: GPTBot
Disallow:

User-agent: anthropic-ai
Disallow:

User-agent: ClaudeBot
Disallow:

User-agent: Sogou
Disallow:

User-agent: Baiduspider
Disallow:

User-agent: Yeti
Disallow:

User-agent: PerplexityBot
Disallow:

# Allow all major social media bots
User-agent: Twitterbot
Disallow:

User-agent: facebookexternalhit
Disallow:

User-agent: LinkedInBot
Disallow:

User-agent: Pinterestbot
Disallow:

User-agent: Slackbot
Disallow:

# General rule for all other compliant bots
User-agent: *
Disallow: /?s=*
Disallow: /search/*
Disallow: /wp-json/
Disallow: /cdn-cgi/bm/cv/
Disallow: /cdn-cgi/challenge-platform/

# Sitemap
Sitemap: https://media.kinkykink.com/sitemap.xml

# Block known malicious, scraping, or non-beneficial bots
User-agent: Nuclei
User-agent: WikiDo
User-agent: Riddler
User-agent: PetalBot
User-agent: Zoominfobot
User-agent: Go-http-client
User-agent: Node/simplecrawler
User-agent: CazoodleBot
User-agent: dotbot/1.0
User-agent: Gigabot
User-agent: Barkrowler
User-agent: BLEXBot
User-agent: magpie-crawler
User-agent: MJ12bot
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: SeznamBot
User-agent: Yandex
User-agent: Exabot
User-agent: Qwantify
User-agent: trendictionbot
User-agent: spbot
User-agent: VelenPublicWebCrawler
Disallow: /