#
# Disallow all user agents
User-agent: *
Disallow: /

# Rate limiting & settings for allowed user agents

User-agent: Twitterbot
User-agent: facebookexternalhit
User-agent: msnbot
User-agent: bingbot
User-agent: Slurp
User-agent: Yahoo! Slurp
User-agent: DuckDuckBot
User-agent: Applebot
Crawl-delay: 1
Disallow:

User-agent: Yandex
Crawl-delay: 4
Disallow:

User-agent: Googlebot
User-agent: Mediapartners-Google
User-agent: AdsBot-Google
User-agent: APIs-Google
User-agent: AdsBot-Google-Mobile
Disallow: