public/robots.txt
# Block the Archive-It bot from crawling (+ storing) our
# repository items, which will be too much data for our budget.
User-agent: archive.org_bot
Disallow: /concern
# Things we don't need to be crawled
User-agent: *
Disallow: /bookmarks/*
Disallow: /catalog
Disallow: /catalog/*
Disallow: /dashboard
Disallow: /dashboard/*
Disallow: /downloads
Disallow: /downloads/*
Disallow: /export/*
Disallow: /iiif/*
Disallow: /redirect/*
# Legacy DSpace paths we no longer support
Disallow: /bitstream
Disallow: /bitstream/*