lib/config/crawlers.yml
crawlers:
addthis: AddThis.com
alexa: ia_archiver
apple: Applebot
archive_org: archive.org_bot
arefs: AhrefsBot
ask: Ask Jeeves
baidu: baidu
bing: bingbot
bitly: bitlybot
blekko: Blekkobot
exabot: Exabot
facebook: facebookexternalhit
flipboard: FlipboardProxy
google: Googlebot
google_web_preview: Google Web Preview
mailru: Mail.RU_Bot
majestic12: MJ12bot
msn: MSNBot
mywebsearch: MyWebSearch
openwebspider: OpenWebSpider
seznam: SeznamBot
technorati: Technoratibot
twitter: Twitterbot
yahoo: Yahoo! Slurp
yahoo_jp: Y!J
yandex: Yandex