# robots.txt - flamicroz.com
# Arendusdomeeni kaitse agressiivsete crawlerite eest

# Faceted search URL-id - lopmatute filtrikombinatsioonide blokeerimine
# See on peamine ressursside uletamise pohjus!
User-agent: *
Disallow: /*?q=
Disallow: /et/avalehele-2?
Disallow: /en/home-2?

# Meta/Facebook crawlerid - taielik blokeering
User-agent: meta-externalagent
Disallow: /

User-agent: meta-webindexer
Disallow: /

# Barkrowler - agressiivne crawler
User-agent: Barkrowler
Disallow: /

# PetalBot (Huawei)
User-agent: PetalBot
Disallow: /

# Bytespider/TikTok
User-agent: Bytespider
Disallow: /

User-agent: TikTokSpider
Disallow: /

# SEO toolid (mittevajalikud arendusdomeenile)
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: IbouBot
Disallow: /

User-agent: serpstatbot
Disallow: /

User-agent: Coccocbot
Disallow: /

User-agent: BitSightBot
Disallow: /

# Crawl-delay pahaaimamatutele botidele
User-agent: Googlebot
Crawl-delay: 10

User-agent: Bingbot
Crawl-delay: 10

User-agent: Applebot
Crawl-delay: 10