bots.yml

360Spider: "360Spider"
AddThis: "AddThis"
ADmantX: "ADmantX Platform Semantic Analyzer"
AdsBot-Google: "Google Adwords"
AhrefsBot: "Ahrefs backlinks research tool"
alexa: "Alexa Crawler"
Apache-HttpClient: "Java http library"
ApacheBench: "ApacheBench (ab)"
AppEngine-Google: "Google App Engine"
Applebot: "Apple Bot"
Ask Jeeves: "Ask Jeeves"
AsyncHttpClient: "Java http and WebSocket client library"
awe.sm: "Awe.sm URL expander"
Baidu: "baidu"
Baiduspider: "Chinese search engine"
bingbot: "Microsoft bing bot"
bitlybot: "bit.ly bot"
BLEXBot: "webmeup-crawler.com"
bot@linkfluence.net: "Linkfluence bot"
BuiBui-CheckBot: "email: buibui[at]dadapro[dot]com"
Butterfly: "Topsy Labs"
buzztalk: "buzztalk"
check_http: "Nagios monitor"
CMRadar/0.1: "CMRadar/0.1"
ColdFusion: "ColdFusion http library"
Crowsnest: "Crowsnest"
Crowsnest: "crowsnest.tv news portal bot"
curl: "curl unix CLI http client"
DAP/NetHTTP: "DAP/NetHTTP"
developers.google.com/+/web/snippet/: "Google Plus"
DigitalPersona Fingerprint Software: "HP Fingerprint scanner"
DotBot: "Dot Bot"
DuckDuckBot: "Duck Duck Go"
Embedly: "Embedly"
EOAAgent: "EOAAgent"
EventMachine HttpClient: "Ruby http library"
EvriNid: "Evri bot"
Exabot: "Exalead's bot"
ExaleadCloudView: "ExaleadCloudView"
facebookexternalhit: "Facebook Bot"
Facebot: "Facebook Bot"
FeedBurner: "RSS bot"
Feedfetcher-Google: "Google Feedfetcher"
FlipboardProxy: "FlipboardProxy"
Genieo: "Genieo Web filter bot"
getprismatic.com: "getprismatic.com"
Gigabot: "Gigabot spider"
Go http package: "Go http library"
Google Web Preview: "Google Instant Previews crawler"
Google Page Speed Insights: "Google Page Speed Insights"
Googlebot: "Google spider"
Hatena::Bookmark: "Hatena::Bookmark"
heritrix: "heritrix"
HTMLParser: "HTMLParser"
HTTPClient: "HTTPClient"
https://developers.google.com/+/web/snippet: "Google+ Snippet Fetcher"
HTTP_Request2: "HTTP_Request2"
HubSpot Connect: "HubSpot Connect"
ia_archiver(OS-Wayback): "ia_archiver(OS-Wayback)"
iCoreService: "iCoreService"
InAGist: "URL resolver"
Insieve Bot: "Insieve Bot"
InsieveBot: "InsieveBot"
Instapaper: "Instapaper"
Insitesbot: "Insitesbot"
IstellaBot: "IstellaBot"
jack: "jack"
Jakarta Commons-HttpClient: "Jakarta Commons-HttpClient"
Jakarta Commons: "Jakarta Commons HttpClient"
Java: "Generic Java http library"
Jetslide: "Jetslide"
JS-Kit: "URL resolver"
Kimengi/nineconnections.com: "Kimengi/nineconnections.com"
Kimengi: "nineconnections.com bot"
knows.is: "knows.is"
kraken: "kraken"
Laconica: "Laconica"
libwww-perl: "Perl client-server library loved by script kids"
linkdexbot: "Linkdex Bot"
LinkedInBot: "LinkedIn"
LinksCrawler: "LinksCrawler"
Linode: "Linode Longview"
Lipperhey: "Lipperhey"
Livelapbot: "Livelapbot"
LoadTimeBot: "Load Time Bot"
LongURL: "URL expander service"
Lumibot: "Lumibot"
lwp-trivial: "Another Perl library loved by script kids"
magpie-crawler: "magpie-crawler"
Mail.RU_Bot: "Mail.ru Bot"
Mediapartners-Google: "Google Adsense bot"
MegaIndex.ru: "Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +https://www.megaindex.ru/?tab=linkAnalyze)"
MetaURI: "MetaURI"
MetaURI: "URI meta information extractor bot"
MFE_expand: "Mcafee spider"
MJ12bot: "Majestic-12 spider"
MojeekBot: "Mojeek UK search crawler"
msnbot: "Microsoft bot"
netState: "netEstate NE Crawler"
Netvibes: "Personalized dashboard bot"
NewRelicPinger: "NewRelic monitor"
newsme: "Link analyzer"
newsme: "newsme"
NING: "NING - Yet Another Twitter Swarmer"
Nutch: "Apache search spider"
Panopta: "Monitoring service"
PaperLiBot: "PaperLi content curation service"
PaperLiBot: "PaperLi is another content curation service"
peerindex: "peerindex"
PercolateCrawler: "PercolateCrawler"
PhantomJS: "PhantomJS"
Pingdom: "Pingdom monitoring"
Pinterest: "Pinterest"
publiclibraryarchive.org: "publiclibraryarchive.org"
PycURL: "Python http library"
Python-httplib2: "Python-httplib2"
python-requests: "Python http library"
python-requests: "python-requests"
Python-urllib: "Python http library"
QuerySeeker: "QuerySeekerSpider ( http://queryseeker.com/bot.html )"
QuickLook: "QuickLook"
Readability: "Readability"
RebelMouse: "RebelMouse"
redditbot: "Reddit Bot"
RelateIQ: "RelateIQ"
Riddler: "Riddler Bot"
rogerbot: "SeoMoz spider"
Ruby/1.9.3: "Ruby/1.9.3"
Ruby: "Ruby"
Scrapy: "Scrapy"
SemrushBot: "SEO analysis bot"
SeznamBot: "SeznamBot"
ShopWiki: "ShopWiki"
ShortLinkTranslate: "Link shortener"
ShowyouBot: "Showyou iOS app spider"
Siege: "Joe Dog Siege"
SiteUptime: "Site monitoring services"
Slack: "Slackbot-LinkExpanding"
Slurp: "Yahoo spider"
Sogou: "Chinese search engine"
spider: "generic web spider"
Spinn3r: "Spinn3r aggregator"
StatusCake: "StatusCake"
Test Certificate Info: "C http library?"
trendictionbot: "trendiction search"
TurnitinBot: "TurnitinBot"
TweetmemeBot: "TweetMeMe Crawler"
Twikle: "Social web search bot"
TwitJobSearch: "TwitJobSearch"
Twitmunin: "Twitmunin"
Twitterbot: "Twitter URL expander"
Twurly: "Twurly"
Typhoeus: "Typhoeus"
UnwindFetch: "Gnip URL expander"
UnwindFetchor: "Gnip crawler"
Vagabondo: "Vagabondo"
VB Project: "Visual Basic"
vkShare: "VKontake Sharer"
VoilaBot: "VoilaBot"
Webscout: "Webscout"
Wget: "wget unix CLI http client"
WordPress: "WordPress spider"
Wormly: "WormlyBot"
Xenu Link Sleuth: "Xenu Link Sleuth"
XoviBot: "XoviBot"
YandexBot: "Yandex spider"
YandexDirect: "Yandex Direct spider"
YandexMetrika: "Yandex Metrika spider"
YOURLS: "YOURLS"
zelist.ro: "feed parser"
ZIBB: "ZIBB spider"
ZyBorg: "Zyborg? Hmmm...."
StructuredDataTestingTool: 'Google-StructuredDataTestingTool'