User-agent: *
Disallow: /register
Disallow: /login
Disallow: /about-us
Disallow: /review-site
Disallow: /add-site
Crawl-Delay: 5

User-Agent: MJ12bot
Crawl-Delay: 20

User-Agent: SemrushBot
Crawl-Delay: 20

User-Agent: SemrushBot/2~bl
Crawl-Delay: 20 
Disallow: /

User-agent: Semrushbot-SA
Crawl-Delay: 20
Disallow: /

User-agent: Semrushbot-SI
Crawl-Delay: 20
Disallow: /

User-agent: HTTrack 3.0 
Disallow: /

User-agent: BecomeBot 
Disallow: / 

User-Agent: YodaoBot
Disallow: /

User-agent: converacrawler
Disallow: /

User-agent: ConveraCrawler
Disallow: /

User-agent: psbot
Disallow: /

User-agent: ichiro
Disallow: /

User-agent: nexabot
Disallow: /

User-agent: voilabot
Disallow: /

User-agent: VoilaBot
Disallow: /

User-agent: yodaobot
Disallow: /

User-agent: twiceler
Disallow: /