public/robots.txt
# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
Sitemap: https://www.planningalerts.org.au/sitemaps/sitemap.xml.gz
# Generally we don't want search pages indexed
User-agent: *
Disallow: /?*q=
# Leaving this commented out because it does appear to get significant
# search traffic
# Disallow: /authorities/*/applications
Disallow: /authorities/*/comments
Allow: /applications/
Disallow: /applications
Disallow: /applications/*/nearby
Disallow: /applications/search
Disallow: /comments
Disallow: /geocode_queries