User-agent: *
Crawl-delay: 5

# No need to index this stuff
Disallow: /api/
Disallow: /connect
Disallow: /forgot
Disallow: /login
Disallow: /profile_loader
Disallow: /redirect
Disallow: /remote_file
Disallow: /review
Disallow: /screenshot
Disallow: /site_review
# Nor this
Disallow: /api/
Disallow: /signup
Disallow: /html/
Disallow: /scripts/
# Avoid duplicate content as it may do more harm than good
Disallow: /*?action=*
Disallow: /*?*&action=*
Disallow: /*?request=*
Disallow: /*?*&request=*
Disallow: /*?sort=*
Disallow: /*?*&sort=*
Disallow: /*?sorting=*
Disallow: /*?*&sorting=*

# Disallow some bots we do not care about
User-agent: 008
Disallow: /
User-agent: AhrefsBot
Disallow: /
User-agent: archive.org_bot
Disallow:/
User-agent: Baiduspider
Disallow: /
User-agent: Barkrowler
Disallow:/
User-agent: BLEXBot
Disallow: /
User-agent: BecomeBot
Disallow: /
User-agent: CCBot
Disallow: /
User-agent: dotbot
Disallow: /
User-agent: EtaoSpider
Disallow: /
User-agent: Exabot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: NPBot
Disallow: /
User-agent: OmniExplorer_Bot
Disallow: /
User-agent: Rogerbot
Disallow: /
User-agent: ShopWiki
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: SemrushBot-BA
Disallow: /
User-agent: SemrushBot-BM
Disallow: /
User-agent: SemrushBot-CT
Disallow: /
User-agent: SemrushBot-SA
Disallow: /
User-agent: SemrushBot-SEOAB
Disallow: /
User-agent: SemrushBot-SI
Disallow: /
User-agent: SemrushBot-SWA
Disallow: /
User-agent: SentiBot
Disallow: /
User-agent: SEOkicks
Disallow: /
User-agent: SMTBot
Disallow: /
User-agent: sogouspider
Disallow:/
User-agent: Sogou web spider
Disallow:/
User-agent: Stanford
Disallow: /
User-agent: Stanford Comp Sci
Disallow: /
User-agent: Synthesio
Disallow: /