public/robots.txt
# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
#
# Not sure why this thing keeps crawling the site
user-agent: AhrefsBot
disallow: /
user-agent: BLEXBot
Disallow: /
# Bots do not need to authenticate
# No downloads
user-agent: *
disallow: /auth/steam
disallow: /auth/allied_modders
disallow: /*.zip