app/views/pages/robots.text.erb
# See https://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
User-agent: *
Disallow: /admin
Disallow: /athletes
Disallow: /volunteers
Disallow: /user/auth/
Disallow: /user/password
Sitemap: <%= File.join root_url, 'sitemap.xml' %>