# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file

User-agent: *
Crawl-delay: 5
Disallow: /*?*page=*
Disallow: /editions/*
Disallow: /users/*
Disallow: /feed
Disallow: /feeds
Disallow: /rss
Disallow: /*?*q=*

User-agent: SemrushBot
Disallow: /

User-agent: PetalBot
Disallow: /