From e917235498aa6672c42c5c55a248e3eb305001d3 Mon Sep 17 00:00:00 2001 From: Andrea Momesso Date: Tue, 5 Feb 2019 15:23:47 +0100 Subject: [PATCH] Bot list now fetched from an updated source --- Rakefile | 2 +- app/models/impressionist/bots.rb | 1463 +-------- lib/botlist/crawler-user-agents.json | 3723 +++++++++++++++++++++++ lib/impressionist/bots.rb | 13 +- tests/test_app/spec/models/bots_spec.rb | 4 +- 5 files changed, 3739 insertions(+), 1466 deletions(-) create mode 100644 lib/botlist/crawler-user-agents.json diff --git a/Rakefile b/Rakefile index c638c0b9..5a2cfba8 100644 --- a/Rakefile +++ b/Rakefile @@ -16,7 +16,7 @@ task :default => [:test, :test_app] namespace :impressionist do require File.dirname(__FILE__) + "/lib/impressionist/bots" - desc "output the list of bots from http://www.user-agents.org/" + desc "output the list of bots from https://github.com/monperrus/crawler-user-agents" task :bots do p Impressionist::Bots.consume end diff --git a/app/models/impressionist/bots.rb b/app/models/impressionist/bots.rb index aee7cf9c..783a4740 100644 --- a/app/models/impressionist/bots.rb +++ b/app/models/impressionist/bots.rb @@ -8,1461 +8,12 @@ def self.bot?(user_agent = nil) WILD_CARDS = ["bot","yahoo","slurp","google","msn","crawler"] - LIST = [" UnChaos From Chaos To Order Hybrid Web Search Engine.(vadim_gonchar@unchaos.com)", - " UnChaos Bot Hybrid Web Search Engine. (vadim_gonchar@unchaos.com)", - " UnChaosBot From Chaos To Order UnChaos Hybrid Web Search Engine at www.unchaos.com (info@unchaos.com)", - " http://www.sygol.com", - "*/Nutch-0.9-dev", - "+SitiDi.net/SitiDiBot/1.0 (+Have Good Day)", - "-DIE-KRAEHE- META-SEARCH-ENGINE/1.1 http://www.die-kraehe.de", - "192.comAgent", - "4anything.com LinkChecker v2.0", - "8484 Boston Project v 1.0", - ":robot/1.0 (linux) ( admin e-mail: undefined http://www.neofonie.de/loesungen/search/robot.html )", - "A-Online Search", - "A1 Sitemap Generator/1.0 (+http://www.micro-sys.dk/products/sitemap-generator/) miggibot/2006.01.24", - "aardvark-crawler", - "AbachoBOT", - "AbachoBOT (Mozilla compatible)", - "ABCdatos BotLink/5.xx.xxx#BBL", - "Aberja Checkomat", - "abot/0.1 (abot; http://www.abot.com; abot@abot.com)", - "About/0.1libwww-perl/5.47", - "Accelatech RSSCrawler/0.4", - "accoona", - "Accoona-AI-Agent/1.1.1 (crawler at accoona dot com)", - "Accoona-AI-Agent/1.1.2 (aicrawler at accoonabot dot com)", - "Ack (http://www.ackerm.com/)", - "AcoiRobot", - "Acoon Robot v1.50.001", - "Acoon Robot v1.52 (http://www.acoon.de)", - "Acoon-Robot 4.0.x.[xx] (http://www.acoon.de)", - "Acoon-Robot v3.xx (http://www.acoon.de and http://www.acoon.com)", - "Acorn/Nutch-0.9 (Non-Profit Search Engine; acorn.isara.org; acorn at isara dot org)", - "AESOP_com_SpiderMan", - "agadine/1.x.x (+http://www.agada.de)", - "Agent-SharewarePlazaFileCheckBot/2.0+(+http://www.SharewarePlaza.com)", - "AgentName/0.1 libwww-perl/5.48", - "AIBOT/2.1 By +(www.21seek.com A Real artificial intelligence search engine China)", - "aipbot/1.0 (aipbot; http://www.aipbot.com; aipbot@aipbot.com)", - "aipbot/2-beta (aipbot dev; http://aipbot.com; aipbot@aipbot.com)", - "Aladin/3.324", - "Aleksika Spider/1.0 (+http://www.aleksika.com/)", - "AlkalineBOT/1.3", - "AlkalineBOT/1.4 (1.4.0326.0 RTM)", - "Allesklar/0.1 libwww-perl/5.46", - "Allrati/1.1 (+)", - "AltaVista Intranet V2.0 AVS EVAL search@freeit.com", - "AltaVista Intranet V2.0 Compaq Altavista Eval sveand@altavista.net", - "AltaVista Intranet V2.0 evreka.com crawler@evreka.com", - "AltaVista V2.0B crawler@evreka.com", - "AmfibiBOT", - "Amfibibot/0.06 (Amfibi Web Search; http://www.amfibi.com; agent@amfibi.com)", - "Amfibibot/0.07 (Amfibi Robot; http://www.amfibi.com; agent@amfibi.com)", - "amibot", - "AnnoMille spider 0.1 alpha - http://www.annomille.it", - "AnswerBus (http://www.answerbus.com/)", - "antibot-V1.1.5/i586-linux-2.2", - "AnzwersCrawl/2.0 (anzwerscrawl@anzwers.com.au;Engine)", - "Apexoo Spider 1.x", - "Aport", - "appie 1.1 (www.walhello.com)", - "ArabyBot (compatible; Mozilla/5.0; GoogleBot; FAST Crawler 6.4; http://www.araby.com;)", - "ArachBot", - "Arachnoidea (arachnoidea@euroseek.com)", - "ArchitextSpider", - "archive.org_bot", - "Arikus_Spider", - "Arquivo-web-crawler (compatible; heritrix/1.12.1 +http://arquivo-web.fccn.pt)", - "ASAHA Search Engine Turkey V.001 (http://www.asaha.com/)", - "Asahina-Antenna/1.x", - "Asahina-Antenna/1.x (libhina.pl/x.x ; libtime.pl/x.x)", - "ask.24x.info", - "AskAboutOil/0.06-rcp (Nutch; http://www.nutch.org/docs/en/bot.html; nutch-agent@askaboutoil.com)", - "asked/Nutch-0.8 (web crawler; http://asked.jp; epicurus at gmail dot com)", - "ASPSeek/1.2.5", - "ASPseek/1.2.9d", - "ASPSeek/1.2.x", - "ASPSeek/1.2.xa", - "ASPseek/1.2.xx", - "ASPSeek/1.2.xxpre", - "ASSORT/0.10", - "asterias/2.0", - "AtlocalBot/1.1 +(http://www.atlocal.com/local-web-site-owner.html)", - "Atomic_Email_Hunter/4.0", - "Atomz/1.0", - "atSpider/1.0", - "Attentio/Nutch-0.9-dev (Attentio's beta blog crawler; www.attentio.com; info@attentio.com)", - "augurfind", - "augurnfind V-1.x", - "autoemailspider", - "autowebdir 1.1 (www.autowebdir.com)", - "AV Fetch 1.0", - "AVSearch-1.0(peter.turney@nrc.ca)", - "AVSearch-3.0(AltaVista/AVC)", - "axadine/ (Axadine Crawler; http://www.axada.de/; )", - "AxmoRobot - Crawling your site for better indexing on www.axmo.com search engine.", - "BabalooSpider/1.3 (BabalooSpider; http://www.babaloo.si; spider@babaloo.si)", - "BaboomBot/1.x.x (+http://www.baboom.us)", - "BaiduImagespider+(+http://www.baidu.jp/search/s308.html)", - "BaiDuSpider", - "Baiduspider+(+http://help.baidu.jp/system/05.html)", - "Baiduspider+(+http://www.baidu.com/search/spider.htm)", - "Baiduspider+(+http://www.baidu.com/search/spider_jp.html)", - "Balihoo/Nutch-1.0-dev (Crawler for Balihoo.com search engine - obeys robots.txt and robots meta tags ; http://balihoo.com/index.aspx; robot at balihoo dot com)", - "BarraHomeCrawler (albertof@barrahome.org)", - "bdcindexer_2.6.2 (research@bdc)", - "BDFetch", - "BDNcentral Crawler v2.3 [en] (http://www.bdncentral.com/robot.html) (X11; I; Linux 2.0.44 i686)", - "beautybot/1.0 (+http://www.uchoose.de/crawler/beautybot/)", - "BebopBot/2.5.1 ( crawler http://www.apassion4jazz.net/bebopbot.html )", - "BigCliqueBOT/1.03-dev (bigclicbot; http://www.bigclique.com; bot@bigclique.com)", - "BIGLOTRON (Beta 2;GNU/Linux)", - "Bigsearch.ca/Nutch-x.x-dev (Bigsearch.ca Internet Spider; http://www.bigsearch.ca/; info@enhancededge.com)", - "BilgiBetaBot/0.8-dev (bilgi.com (Beta) ; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)", - "BilgiBot/1.0(beta) (http://www.bilgi.com/; bilgi at bilgi dot com)", - "Bitacle bot/1.1", - "Bitacle Robot (V:1.0;) (http://www.bitacle.com)", - "BlackWidow", - "Blaiz-Bee/1.0 (+http://www.blaiz.net)", - "Blaiz-Bee/2.00.8222 (BE Internet Search Engine http://www.rawgrunt.com)", - "Blaiz-Bee/2.00.xxxx (+http://www.blaiz.net)", - "BlitzBOT@tricus.net", - "BlitzBOT@tricus.net (Mozilla compatible)", - "BlogBot/1.x", - "Bloglines Title Fetch/1.0 (http://www.bloglines.com)", - "Bloglines-Images/0.1 (http://www.bloglines.com)", - "Bloglines/3.1 (http://www.bloglines.com)", - "Blogpulse (info@blogpulse.com)", - "BlogPulseLive (support@blogpulse.com)", - "BlogSearch/1.x +http://www.icerocket.com/", - "blogsearchbot-pumpkin-3", - "BlogsNowBot, V 2.01 (+http://www.blogsnow.com/)", - "BlogVibeBot-v1.1 (spider@blogvibe.nl)", - "blogWatcher_Spider/0.1 (http://www.lr.pi.titech.ac.jp/blogWatcher/)", - "BlogzIce/1.0 (+http://icerocket.com; rhodes@icerocket.com)", - "BlogzIce/1.0 +http://www.icerocket.com/", - "BloobyBot", - "Bloodhound/Nutch-0.9 (Testing Crawler for Research - obeys robots.txt and robots meta tags ; http://balihoo.com/index.aspx; robot at balihoo dot com)", - "boitho.com-dc/0.xx (http://www.boitho.com/dcbot.html)", - "boitho.com-robot/1.x", - "boitho.com-robot/1.x (http://www.boitho.com/bot.html)", - "BPImageWalker/2.0 (www.bdbrandprotect.com)", - "BravoBrian SpiderEngine MarcoPolo", - "BruinBot (+http://webarchive.cs.ucla.edu/bruinbot.html) ", - "BSDSeek/1.0", - "BTbot/0.x (+http://www.btbot.com/btbot.html)", - "BuildCMS crawler (http://www.buildcms.com/crawler)", - "BullsEye", - "bumblebee@relevare.com", - "BurstFindCrawler/1.1 (crawler.burstfind.com; http://crawler.burstfind.com; crawler@burstfind.com)", - "Buscaplus Robi/1.0 (http://www.buscaplus.com/robi/)", - "bwh3_user_agent", - "Cabot/Nutch-0.9 (Amfibi's web-crawling robot; http://www.amfibi.com/cabot/; agent@amfibi.com)", - "Cabot/Nutch-1.0-dev (Amfibi's web-crawling robot; http://www.amfibi.com/cabot/; agent@amfibi.com)", - "carleson/1.0", - "Carnegie_Mellon_University_Research_WebBOT-->PLEASE READ-->http://www.andrew.cmu.edu/~brgordon/webbot/index.html http://www.andrew.cmu.edu/~brgordon/webbot/index.html", - "Carnegie_Mellon_University_WebCrawler http://www.andrew.cmu.edu/~brgordon/webbot/index.html", - "Catall Spider", - "CazoodleBot/CazoodleBot-0.1 (CazoodleBot Crawler; http://www.cazoodle.com/cazoodlebot; cazoodlebot@cazoodle.com)", - "CCBot/1.0 (+http://www.commoncrawl.org/bot.html)", - "ccubee/x.x", - "Ceramic Tile Installation Guide (http://www.floorstransformed.com)", - "cfetch/1.0", - "China Local Browse 2.6", - "ChristCRAWLER 2.0", - "CipinetBot (http://www.cipinet.com/bot.html)", - "ClariaBot/1.0", - "Claymont.com", - "CloakDetect/0.9 (+http://fulltext.seznam.cz/)", - "Clushbot/2.x (+http://www.clush.com/bot.html)", - "Clushbot/3.x-BinaryFury (+http://www.clush.com/bot.html)", - "Clushbot/3.xx-Ajax (+http://www.clush.com/bot.html)", - "Clushbot/3.xx-Hector (+http://www.clush.com/bot.html)", - "Clushbot/3.xx-Peleus (+http://www.clush.com/bot.html)", - "Cogentbot/1.X (+http://www.cogentsoftwaresolutions.com/bot.html)", - "combine/0.0", - "Combine/2.0 http://combine.it.lth.se/", - "Combine/3 http://combine.it.lth.se/", - "Combine/x.0", - "cometrics-bot, http://www.cometrics.de", - "Computer_and_Automation_Research_Institute_Crawler crawler@ilab.sztaki.hu", - "Comrite/0.7.1 (Nutch; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)", - "ContactBot/0.2", - "ContentSmartz", - "Convera Internet Spider V6.x", - "ConveraCrawler/0.2", - "ConveraCrawler/0.9d (+http://www.authoritativeweb.com/crawl)", - "ConveraMultiMediaCrawler/0.1 (+http://www.authoritativeweb.com/crawl)", - "CoolBot", - "cosmos/0.8_(robot@xyleme.com)", - "cosmos/0.9_(robot@xyleme.com)", - "CougarSearch/0.x (+http://www.cougarsearch.com/faq.shtml)", - "Covac TexAs Arachbot", - "Cowbot-0.1 (NHN Corp. / +82-2-3011-1954 / nhnbot@naver.com)", - "Cowbot-0.1.x (NHN Corp. / +82-2-3011-1954 / nhnbot@naver.com)", - "CrawlConvera0.1 (CrawlConvera@yahoo.com)", - "Crawler (cometsearch@cometsystems.com)", - "Crawler admin@crawler.de", - "Crawler V 0.2.x admin@crawler.de", - "crawler@alexa.com", - "CrawlerBoy Pinpoint.com", - "Crawllybot/0.1 (Crawllybot; +http://www.crawlly.com; crawler@crawlly.com)", - "CreativeCommons/0.06-dev (Nutch; http://www.nutch.org/docs/en/bot.html; nutch-agent@lists.sourceforge.net)", - "CrocCrawler vx.3 [en] (http://www.croccrawler.com) (X11; I; Linux 2.0.44 i686)", - "csci_b659/0.13", - "Cuasarbot/0.9b http://www.cuasar.com/spider_beta/ ", - "CurryGuide SiteScan 1.1", - "Custom Spider www.bisnisseek.com /1.0", - "CyberPatrol SiteCat Webbot (http://www.cyberpatrol.com/cyberpatrolcrawler.asp)", - "CydralSpider/1.x (Cydral Web Image Search; http://www.cydral.com)", - "CydralSpider/3.0 (Cydral Image Search; http://www.cydral.com)", - "DataCha0s/2.0", - "DataCha0s/2.0", - "DataFountains/DMOZ Downloader", - "DataFountains/Dmoz Downloader (http://ivia.ucr.edu/useragents.shtml)", - "DataFountains/DMOZ Feature Vector Corpus Creator (http://ivia.ucr.edu/useragents.shtml)", - "DataparkSearch/4.47 (+http://dataparksearch.org/bot)", - "DataparkSearch/4.xx (http://www.dataparksearch.org/)", - "DataSpear/1.0 (Spider; http://www.dataspear.com/spider.html; spider@dataspear.com)", - "DataSpearSpiderBot/0.2 (DataSpear Spider Bot; http://dssb.dataspear.com/bot.html; dssb@dataspear.com)", - "DatenBot( http://www.sicher-durchs-netz.de/bot.html)", - "DaviesBot/1.7 (www.wholeweb.net)", - "daypopbot/0.x", - "dbDig(http://www.prairielandconsulting.com)", - "DBrowse 1.4b", - "DBrowse 1.4d", - "dCSbot/1.1", - "de.searchengine.comBot 1.2 (http://de.searchengine.com/spider)", - "deepak-USC/ISI", - "DeepIndex", - "DeepIndex ( http://www.zetbot.com )", - "DeepIndex (www.en.deepindex.com)", - "DeepIndexer.ca", - "Demo Bot DOT 16b", - "Demo Bot Z 16b", - "Denmex websearch (http://search.denmex.com)", - "dev-spider2.searchpsider.com/1.3b", - "DiaGem/1.1 (http://www.skyrocket.gr.jp/diagem.html)", - "Diamond/x.0", - "DiamondBot", - "Digger/1.0 JDK/1.3.0rc3", - "DigOut4U", - "DIIbot/1.2", - "disco/Nutch-0.9 (experimental crawler; www.discoveryengine.com; disco-crawl@discoveryengine.com)", - "disco/Nutch-1.0-dev (experimental crawler; www.discoveryengine.com; disco-crawl@discoveryengine.com)", - "DittoSpyder", - "dloader(NaverRobot)/1.0", - "DoCoMo/1.0/Nxxxi/c10", - "DoCoMo/1.0/Nxxxi/c10/TB", - "DoCoMo/2.0 P900iV(c100;TB;W24H11) ", - "DoCoMo/2.0 SH902i (compatible; Y!J-SRD/1.0; http://help.yahoo.co.jp/help/jp/search/indexing/indexing-27.html)", - "DoCoMo/2.0/SO502i (compatible; Y!J-SRD/1.0; http://help.yahoo.co.jp/help/jp/search/indexing/indexing-27.html)", - "dodgebot/experimental", - "Download-Tipp Linkcheck (http://download-tipp.de/)", - "Drecombot/1.0 (http://career.drecom.jp/bot.html)", - "DSurf15a 01", - "DSurf15a 71", - "DSurf15a 81", - "DSurf15a VA", - "dtSearchSpider", - "DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)", - "Dumbot(version 0.1 beta - dumbfind.com)", - "Dumbot(version 0.1 beta - http://www.dumbfind.com/dumbot.html)", - "Dumbot(version 0.1 beta)", - "e-sense 1.0 ea(www.vigiltech.com/esensedisclaim.html)", - "e-SocietyRobot(http://www.yama.info.waseda.ac.jp/~yamana/es/)", - "eApolloBot/2.0 (compatible; heritrix/2.0.0-SNAPSHOT-20071024.170148 +http://www.eapollo-opto.com)", - "EARTHCOM.info/1.x [www.earthcom.info]", - "EARTHCOM.info/1.xbeta [www.earthcom.info]", - "EasyDL/3.xx", - "EasyDL/3.xx http://keywen.com/Encyclopedia/Bot", - "EBrowse 1.4b", - "EchO!/2.0", - "Educate Search VxB", - "egothor/3.0a (+http://www.xdefine.org/robot.html)", - "EgotoBot/4.8 (+http://www.egoto.com/about.htm)", - "ejupiter.com", - "elfbot/1.0 (+http://www.uchoose.de/crawler/elfbot/)", - "ELI/20070402:2.0 (DAUM RSS Robot, Daum Communications Corp.; +http://ws.daum.net/aboutkr.html)", - "EmailSiphon", - "EmailSpider", - "EmailWolf 1.00", - "EMPAS_ROBOT", - "EnaBot/1.x (http://www.enaball.com/crawler.html)", - "Enfish Tracker", - "Enterprise_Search/1.0", - "Enterprise_Search/1.0.xxx", - "Enterprise_Search/1.00.xxx;MSSQL (http://www.innerprise.net/es-spider.asp)", - "envolk/1.7 (+http://www.envolk.com/envolkspiderinfo.php)", - "envolk[ITS]spider/1.6(+http://www.envolk.com/envolkspider.html)", - "EroCrawler", - "ES.NET_Crawler/2.0 (http://search.innerprise.net/)", - "eseek-larbin_2.6.2 (crawler@exactseek.com)", - "ESISmartSpider", - "eStyleSearch 4 (compatible; MSIE 6.0; Windows NT 5.0)", - "ESurf15a 15", - "EuripBot/0.x (+http://www.eurip.com) GetFile", - "EuripBot/0.x (+http://www.eurip.com) GetRobots", - "EuripBot/0.x (+http://www.eurip.com) PreCheck", - "Eurobot/1.0 (http://www.ayell.eu)", - "EvaalSE - bot@evaal.com", - "eventax/1.3 (eventax; http://www.eventax.de/; info@eventax.de)", - "Everest-Vulcan Inc./0.1 (R&D project; host=e-1-24; http://everest.vulcan.com/crawlerhelp)", - "Everest-Vulcan Inc./0.1 (R&D project; http://everest.vulcan.com/crawlerhelp)", - "Exabot-Images/1.0", - "Exabot-Test/1.0", - "Exabot/2.0", - "Exabot/3.0", - "ExactSeek Crawler/0.1", - "exactseek-crawler-2.63 (crawler@exactseek.com)", - "exactseek-pagereaper-2.63 (crawler@exactseek.com)", - "exactseek.com", - "Exalead NG/MimeLive Client (convert/http/0.120)", - "Excalibur Internet Spider V6.5.4", - "Execrawl/1.0 (Execrawl; http://www.execrawl.com/; bot@execrawl.com)", - "exooba crawler/exooba crawler (crawler for exooba.com; http://www.exooba.com/; info at exooba dot com)", - "exooba/exooba crawler (exooba; exooba)", - "ExperimentalHenrytheMiragoRobot", - "ExtractorPro", - "EyeCatcher (Download-tipp.de)/1.0", - "Factbot 1.09 (see http://www.factbites.com/webmasters.php)", - "factbot : http://www.factbites.com/robots", - "Fast Crawler Gold Edition", - "FAST Enterprise Crawler 6 (Experimental)", - "FAST Enterprise Crawler 6 / Scirus scirus-crawler@fast.no; http://www.scirus.com/srsapp/contactus/", - "FAST Enterprise Crawler 6 used by Cobra Development (admin@fastsearch.com)", - "FAST Enterprise Crawler 6 used by Comperio AS (sts@comperio.no)", - "FAST Enterprise Crawler 6 used by FAST (FAST)", - "FAST Enterprise Crawler 6 used by Pages Jaunes (pvincent@pagesjaunes.fr)", - "FAST Enterprise Crawler 6 used by Sensis.com.au Web Crawler (search_comments\\at\\sensis\\dot\\com\\dot\\au)", - "FAST Enterprise Crawler 6 used by Singapore Press Holdings (crawler@sphsearch.sg)", - "FAST Enterprise Crawler/6 (www.fastsearch.com)", - "FAST Enterprise Crawler/6.4 (helpdesk at fast.no)", - "FAST FirstPage retriever (compatible; MSIE 5.5; Mozilla/4.0)", - "FAST MetaWeb Crawler (helpdesk at fastsearch dot com)", - "Fast PartnerSite Crawler", - "FAST-WebCrawler/2.2.10 (Multimedia Search) (crawler@fast.no; http://www.fast.no/faq/faqfastwebsearch/faqfastwebcrawler.html)", - "FAST-WebCrawler/2.2.6 (crawler@fast.no; http://www.fast.no/faq/faqfastwebsearch/faqfastwebcrawler.html)", - "FAST-WebCrawler/2.2.7 (crawler@fast.no; http://www.fast.no/faq/faqfastwebsearch/faqfastwebcrawler.html)http://www.fast.no", - "FAST-WebCrawler/2.2.8 (crawler@fast.no; http://www.fast.no/faq/faqfastwebsearch/faqfastwebcrawler.html)http://www.fast.no", - "FAST-WebCrawler/3.2 test", - "FAST-WebCrawler/3.3 (crawler@fast.no; http://fast.no/support.php?c=faqs/crawler)", - "FAST-WebCrawler/3.4/Nirvana (crawler@fast.no; http://fast.no/support.php?c=faqs/crawler)", - "FAST-WebCrawler/3.4/PartnerSite (crawler@fast.no; http://fast.no/support.php?c=faqs/crawler)", - "FAST-WebCrawler/3.5 (atw-crawler at fast dot no; http://fast.no/support.php?c=faqs/crawler)", - "FAST-WebCrawler/3.6 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)", - "FAST-WebCrawler/3.6/FirstPage (crawler@fast.no; http://fast.no/support.php?c=faqs/crawler)", - "FAST-WebCrawler/3.7 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)", - "FAST-WebCrawler/3.7/FirstPage (atw-crawler at fast dot no;http://fast.no/support/crawler.asp)", - "FAST-WebCrawler/3.8 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)", - "FAST-WebCrawler/3.8/Fresh (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)", - "FAST-WebCrawler/3.x Multimedia", - "FAST-WebCrawler/3.x Multimedia (mm dash crawler at fast dot no)", - "fastbot crawler beta 2.0 (+http://www.fastbot.de)", - "FastBug http://www.ay-up.com", - "FastCrawler 3.0.1 (crawler@1klik.dk)", - "FastSearch Web Crawler for Verizon SuperPages (kevin.watters@fastsearch.com)", - "Favcollector/2.0 (info@favcollector.com http://www.favcollector.com/)", - "favo.eu crawler/0.6 (http://www.favo.eu)", - "Faxobot/1.0", - "Feed Seeker Bot (RSS Feed Seeker http://www.MyNewFavoriteThing.com/fsb.php)", - "Feed24.com", - "FeedChecker/0.01", - "Feedfetcher-Google; (+http://www.google.com/feedfetcher.html)", - "FeedHub FeedDiscovery/1.0 (http://www.feedhub.com)", - "FeedHub MetaDataFetcher/1.0 (http://www.feedhub.com)", - "Feedjit Favicon Crawler 1.0", - "Feedster Crawler/3.0; Feedster, Inc.", - "Felix - Mixcat Crawler (+http://mixcat.com)", - "FFC Trap Door Spider", - "Filtrbox/1.0", - "Findexa Crawler (http://www.findexa.no/gulesider/article26548.ece)", - "findlinks/x.xxx (+http://wortschatz.uni-leipzig.de/findlinks/) ", - "FineBot", - "Firefly/1.0", - "Firefly/1.0 (compatible; Mozilla 4.0; MSIE 5.5)", - "Firefox (kastaneta03@hotmail.com)", - "Firefox_1.0.6 (kasparek@naparek.cz)", - "FirstGov.gov Search - POC:firstgov.webmasters@gsa.gov", - "firstsbot", - "Flapbot/0.7.2 (Flaptor Crawler; http://www.flaptor.com; crawler at flaptor period com)", - "Flexum spider", - "Flexum/2.0", - "FlickBot 2.0 RPT-HTTPClient/0.3-3", - "flunky", - "FnooleBot/2.5.2 (+http://www.fnoole.com/addurl.html)", - "FocusedSampler/1.0", - "Folkd.com Spider/0.1 beta 1 (www.folkd.com)", - "Fooky.com/ScorpionBot/ScoutOut; http://www.fooky.com/scorpionbots", - "Francis/1.0 (francis@neomo.de http://www.neomo.de/)", - "Franklin Locator 1.8", - "FreeFind.com-SiteSearchEngine/1.0 (http://freefind.com; spiderinfo@freefind.com)", - "FreshNotes crawler< report problems to crawler-at-freshnotes-dot-com", - "FSurf15a 01", - "FTB-Bot http://www.findthebest.co.uk/", - "Full Web Bot 0416B", - "Full Web Bot 0516B", - "Full Web Bot 2816B", - "FuseBulb.Com", - "FyberSpider (+http://www.fybersearch.com/fyberspider.php)", - "GAIS Robot/1.0B2", - "Gaisbot/3.0 (indexer@gais.cs.ccu.edu.tw; http://gais.cs.ccu.edu.tw/robot.php)", - "Gaisbot/3.0+(robot06@gais.cs.ccu.edu.tw;+http://gais.cs.ccu.edu.tw/robot.php)", - "GalaxyBot/1.0 (http://www.galaxy.com/galaxybot.html)", - "Gallent Search Spider v1.4 Robot 2 (http://robot.GallentSearch.com)", - "gamekitbot/1.0 (+http://www.uchoose.de/crawler/gamekitbot/)", - "GammaSpider/1.0", - "gazz/x.x (gazz@nttrd.com)", - "generic_crawler/01.0217/", - "genieBot (http://64.5.245.11/faq/faq.html)", - "geniebot wgao@genieknows.com", - "GeonaBot 1.x; http://www.geona.com/", - "gigabaz/3.1x (baz@gigabaz.com; http://gigabaz.com/gigabaz/)", - "Gigabot/2.0 (gigablast.com)", - "Gigabot/2.0/gigablast.com/spider.html", - "Gigabot/2.0; http://www.gigablast.com/spider.html", - "Gigabot/2.0att", - "Gigabot/3.0 (http://www.gigablast.com/spider.html)", - "Gigabot/x.0", - "GigabotSiteSearch/2.0 (sitesearch.gigablast.com)", - "GNODSPIDER (www.gnod.net)", - "Goblin/0.9 (http://www.goguides.org/)", - "Goblin/0.9.x (http://www.goguides.org/goblin-info.html)", - "GoForIt.com", - "GOFORITBOT ( http://www.goforit.com/about/ )", - "gonzo1[P] +http://www.suchen.de/popups/faq.jsp", - "gonzo2[P] +http://www.suchen.de/faq.html", - "Goofer/0.2", - "Googlebot-Image/1.0", - "Googlebot-Image/1.0 ( http://www.googlebot.com/bot.html)", - "Googlebot/2.1 ( http://www.google.com/bot.html)", - "Googlebot/2.1 ( http://www.googlebot.com/bot.html)", - "Googlebot/Test ( http://www.googlebot.com/bot.html)", - "GrapeFX/0.3 libwww/5.4.0", - "great-plains-web-spider/flatlandbot (Flatland Industries Web Spider; http://www.flatlandindustries.com/flatlandbot.php; jason@flatlandindustries.com)", - "GrigorBot 0.8 (http://www.grigor.biz/bot.html)", - "Gromit/1.0", - "grub crawler(http://www.grub.org)", - "grub-client", - "gsa-crawler (Enterprise; GID-01422; jplastiras@google.com)", - "gsa-crawler (Enterprise; GID-01742;gsatesting@rediffmail.com)", - "gsa-crawler (Enterprise; GIX-02057; dm@enhesa.com)", - "gsa-crawler (Enterprise; GIX-03519; cknuetter@stubhub.com)", - "gsa-crawler (Enterprise; GIX-0xxxx; enterprise-training@google.com)", - "Guestbook Auto Submitter", - "Gulliver/1.3", - "Gulper Web Bot 0.2.4 (www.ecsl.cs.sunysb.edu/~maxim/cgi-bin/Link/GulperBot)", - "Gungho/0.08004 (http://code.google.com/p/gungho-crawler/wiki/Index)", - "GurujiBot/1.0 (+http://www.guruji.com/WebmasterFAQ.html)", - "GurujiImageBot/1.0 (+http://www.guruji.com/en/WebmasterFAQ.html)", - "HappyFunBot/1.1", - "Harvest-NG/1.0.2", - "Hatena Antenna/0.4 (http://a.hatena.ne.jp/help#robot)", - "Hatena Pagetitle Agent/1.0", - "Hatena RSS/0.3 (http://r.hatena.ne.jp)", - "hbtronix.spider.2 -- http://hbtronix.de/spider.php", - "HeinrichderMiragoRobot", - "HeinrichderMiragoRobot (http://www.miragorobot.com/scripts/deinfo.asp)", - "Helix/1.x ( http://www.sitesearch.ca/helix/)", - "HenriLeRobotMirago (http://www.miragorobot.com/scripts/frinfo.asp)", - "HenrytheMiragoRobot", - "HenryTheMiragoRobot (http://www.miragorobot.com/scripts/mrinfo.asp)", - "Hi! I'm CsCrawler my homepage: http://www.kde.cs.uni-kassel.de/lehre/ss2005/googlespam/crawler.html RPT-HTTPClient/0.3-3", - "Hippias/0.9 Beta", - "HitList", - "Hitwise Spider v1.0 http://www.hitwise.com", - "holmes/3.11 (http://morfeo.centrum.cz/bot)", - "holmes/3.9 (onet.pl)", - "holmes/3.xx (OnetSzukaj/5.0; +http://szukaj.onet.pl)", - "holmes/x.x", - "HolmesBot (http://holmes.ge)", - "HomePageSearch(hpsearch.uni-trier.de)", - "Homerbot: www.homerweb.com", - "Honda-Search/0.7.2 (Nutch; http://lucene.apache.org/nutch/bot.html; search@honda-search.com)", - "HooWWWer/2.1.3 (debugging run) (+http://cosco.hiit.fi/search/hoowwwer/ | mailto:crawler-infohiit.fi)", - "HooWWWer/2.1.x ( http://cosco.hiit.fi/search/hoowwwer/ | mailto:crawler-infohiit.fi)", - "HPL/Nutch-0.9 -", - "htdig/3.1.6 (http://computerorgs.com)", - "htdig/3.1.6 (unconfigured@htdig.searchengine.maintainer)", - "htdig/3.1.x (root@localhost)", - "http://Ask.24x.Info/ (http://narres.it/)", - "http://hilfe.acont.de/bot.html ACONTBOT", - "http://www.almaden.ibm.com/cs/crawler", - "http://www.almaden.ibm.com/cs/crawler [rc1.wf.ibm.com]", - "http://www.almaden.ibm.com/cs/crawler [wf216]", - "http://www.istarthere.com_spider@istarthere.com", - "http://www.monogol.de", - "http://www.trendtech.dk/spider.asp)", - "i1searchbot/2.0 (i1search web crawler; http://www.i1search.com; crawler@i1search.com)", - "IAArchiver-1.0", - "iaskspider2 (iask@staff.sina.com.cn)", - "ia_archiver", - "ia_archiver-web.archive.org", - "ia_archiver/1.6", - "ICC-Crawler(Mozilla-compatible; http://kc.nict.go.jp/icc/crawl.html; icc-crawl(at)ml(dot)nict(dot)go(dot)jp)", - "ICC-Crawler(Mozilla-compatible;http://kc.nict.go.jp/icc/crawl.html;icc-crawl-contact(at)ml(dot)nict(dot)go(dot)jp)", - "iCCrawler (http://www.iccenter.net)", - "ICCrawler - ICjobs (http://www.icjobs.de/bot.htm)", - "ichiro/x.0 (http://help.goo.ne.jp/door/crawler.html)", - "ichiro/x.0 (ichiro@nttr.co.jp)", - "IconSurf/2.0 favicon finder (see http://iconsurf.com/robot.html)", - "IconSurf/2.0 favicon monitor (see http://iconsurf.com/robot.html)", - "ICRA_label_spider/x.0", - "icsbot-0.1", - "ideare - SignSite/1.x", - "iFeed.jp/2.0 (www.psychedelix.com/agents/agents.rss; 0 subscribers)", - "igdeSpyder (compatible; igde.ru; +http://igde.ru/doc/tech.html)", - "IIITBOT/1.1 (Indian Language Web Search Engine; http://webkhoj.iiit.net; pvvpr at iiit dot ac dot in)", - "ilial/Nutch-0.9 (Ilial, Inc. is a Los Angeles based Internet startup company. For more information please visit http://www.ilial.com/crawler; http://www.ilial.com/crawler; crawl@ilial.com)", - "ilial/Nutch-0.9-dev", - "IlseBot/1.x", - "IlTrovatore-Setaccio ( http://www.iltrovatore.it)", - "Iltrovatore-Setaccio/0.3-dev (Indexing; http://www.iltrovatore.it/bot.html; info@iltrovatore.it)", - "IlTrovatore-Setaccio/1.2 ( http://www.iltrovatore.it/aiuto/faq.html)", - "Iltrovatore-Setaccio/1.2 (It-bot; http://www.iltrovatore.it/bot.html; info@iltrovatore.it)", - "iltrovatore-setaccio/1.2-dev (spidering; http://www.iltrovatore.it/aiuto/.....)", - "IlTrovatore/1.2 (IlTrovatore; http://www.iltrovatore.it/bot.html; bot@iltrovatore.it)", - "ImageWalker/2.0 (www.bdbrandprotect.com)", - "IncyWincy data gatherer(webmaster@loopimprovements.com", - "IncyWincy page crawler(webmaster@loopimprovements.com", - "IncyWincy(http://www.look.com)", - "IncyWincy(http://www.loopimprovements.com/robot.html)", - "IncyWincy/2.1(loopimprovements.com/robot.html)", - "IndexTheWeb.com Crawler7", - "Industry Program 1.0.x", - "Inet library", - "info@pubblisito.com- (http://www.pubblisito.com) il Sud dei Motori di Ricerca", - "InfoFly/1.0 (http://www.versions-project.org/)", - "INFOMINE/8.0 Adders", - "INFOMINE/8.0 RemoteServices", - "INFOMINE/8.0 VLCrawler (http://infomine.ucr.edu/useragents)", - "InfoNaviRobot(F107)", - "InfoSeek Sidewinder/0.9", - "InfoSeek Sidewinder/1.0A", - "InfoSeek Sidewinder/1.1A", - "Infoseek SideWinder/1.45 (Compatible; MSIE 10.0; UNIX)", - "Infoseek SideWinder/2.0B (Linux 2.4 i686)", - "INGRID/3.0 MT (webcrawler@NOSPAMexperimental.net; http://webmaster.ilse.nl/jsp/webmaster.jsp)", - "Inktomi Search", - "InnerpriseBot/1.0 (http://www.innerprise.com/)", - "Insitor.com search and find world wide!", - "Insitornaut", - "Internet Ninja x.0", - "InternetArchive/0.8-dev(Nutch;http://lucene.apache.org/nutch/bot.html;nutch-agent@lucene.apache", - "InternetSeer.com", - "IOI/2.0 (ISC Open Index crawler; http://index.isc.org/; bot@index.isc.org)", - "IPiumBot laurion(dot)com", - "IpselonBot/0.xx-beta (Ipselon; http://www.ipselon.com; ipselonbot@ipselon.com)", - "IRLbot/1.0 ( http://irl.cs.tamu.edu/crawler)", - "IRLbot/3.0 (compatible; MSIE 6.0; http://irl.cs.tamu.edu/crawler/)", - "ISC Systems iRc Search 2.1", - "IUPUI Research Bot v 1.9a", - "IWAgent/ 1.0 - www.brandprotect.com", - "Jabot/6.x (http://odin.ingrid.org/)", - "Jabot/7.x.x (http://odin.ingrid.org/)", - "Jack", - "Jambot/0.1.x (Jambot; http://www.jambot.com/blog; crawler@jambot.com)", - "Jambot/0.2.1 (Jambot; http://www.jambot.com/blog/static.php?page=webmaster-robot; crawler@jambot.com)", - "Jayde Crawler. http://www.jayde.com", - "Jetbot/1.0", - "JobSpider_BA/1.1", - "Jyxobot/x", - "k2spider", - "KAIST AITrc Crawler", - "KakleBot - www.kakle.com/0.1 (KakleBot - www.kakle.com; http:// www.kakle.com/bot.html; support@kakle.com)", - "kalooga/kalooga-4.0-dev-datahouse (Kalooga; http://www.kalooga.com; info@kalooga.com)", - "kalooga/KaloogaBot (Kalooga; http://www.kalooga.com/info.html?page=crawler; crawler@kalooga.com)", - "Kenjin Spider", - "Kevin http://dznet.com/kevin/", - "Kevin http://websitealert.net/kevin/", - "KE_1.0/2.0 libwww/5.2.8", - "KFSW-Bot (Version: 1.01 powered by KFSW www.kfsw.de)", - "kinja-imagebot (http://www.kinja.com/)", - "kinjabot (http://www.kinja.com)", - "KIT-Fireball/2.0", - "KIT-Fireball/2.0 (compatible; Mozilla 4.0; MSIE 5.5)", - "KnowItAll(knowitall@cs.washington.edu)", - "Knowledge.com/0.x", - "Krugle/Krugle,Nutch/0.8+ (Krugle web crawler; http://www.krugle.com/crawler/info.html; webcrawler@krugle.com)", - "KSbot/1.0 (KnowledgeStorm crawler; http://www.knowledgestorm.com/resources/content/crawler/index.html; crawleradmin@knowledgestorm.com)", - "kuloko-bot/0.x", - "kulokobot www.kuloko.com kuloko@backweave.com", - "kulturarw3/0.1", - "LapozzBot/1.4 ( http://robot.lapozz.com)", - "LapozzBot/1.5 (+http://robot.lapozz.hu)", - "larbin (samualt9@bigfoot.com)", - "LARBIN-EXPERIMENTAL (efp@gmx.net)", - "larbin_2.1.1 larbin2.1.1@somewhere.com", - "larbin_2.2.0 (crawl@compete.com)", - "larbin_2.2.1_de_Viennot (Laurent.Viennot@inria.fr)", - "larbin_2.2.2 (sugayama@lab7.kuis.kyoto-u.ac.jp)", - "larbin_2.2.2_guillaume (guillaume@liafa.jussieu.fr)", - "larbin_2.6.0 (larbin2.6.0@unspecified.mail)", - "larbin_2.6.1 (larbin2.6.1@unspecified.mail)", - "larbin_2.6.2 (hamasaki@grad.nii.ac.jp)", - "larbin_2.6.2 (larbin2.6.2@unspecified.mail)", - "larbin_2.6.2 (listonATccDOTgatechDOTedu)", - "larbin_2.6.2 (pimenas@systems.tuc.gr)", - "larbin_2.6.2 (tom@lemurconsulting.com)", - "larbin_2.6.2 (vitalbox1@hotmail.com)", - "larbin_2.6.3 (ltaa_web_crawler@groupes.epfl.ch)", - "larbin_2.6.3 (wgao@genieknows.com)", - "larbin_2.6.3_for_(http://cosco.hiit.fi/search/) tsilande@hiit.fi", - "larbin_2.6_basileocaml (basile.starynkevitch@cea.fr)", - "larbin_devel (http://pauillac.inria.fr/~ailleret/prog/larbin/)", - "lawinfo-crawler/Nutch-0.9-dev (Crawler for lawinfo.com pages; http://www.lawinfo.com; webmaster@lawinfo.com)", - "LECodeChecker/3.0 libgetdoc/1.0", - "LEIA/2.90", - "LEIA/3.01pr (LEIAcrawler; [SNIP])", - "LetsCrawl.com/1.0 +http://letscrawl.com/", - "LexiBot/1.00", - "Libby_1.1/libwww-perl/5.47", - "LibertyW (+http://www.lw01.com)", - "libWeb/clsHTTP -- hiongun@kt.co.kr", - "libwww-perl/5.41", - "libwww-perl/5.45", - "libwww-perl/5.48", - "libwww-perl/5.52 FP/2.1", - "libwww-perl/5.52 FP/4.0", - "libwww-perl/5.65", - "libwww-perl/5.800", - "libwww/5.3.2", - "LijitSpider/Nutch-0.9 (Reports crawler; http://www.lijit.com/; info(a)lijit(d)com)", - "Lincoln State Web Browser", - "linkbot", - "linknzbot", - "Links 2.0 (http://gossamer-threads.com/scripts/links/)", - "Links SQL (http://gossamer-threads.com/scripts/links-sql/)", - "LinkScan/11.0beta2 UnixShareware robot from Elsop.com (used by Indiafocus/Indiainfo)", - "LinkScan/9.0g Unix", - "LinkScan/x.x Unix", - "LiveTrans/Nutch-0.9 (maintainer: cobain at iis dot sinica dot edu dot tw; http://wkd.iis.sinica.edu.tw/LiveTrans/)", - "Llaut/1.0 (http://mnm.uib.es/~gallir/llaut/bot.html)", - "LMQueueBot/0.2", - "lmspider (lmspider@scansoft.com)", - "LNSpiderguy", - "LocalBot/1.0 ( http://www.localbot.co.uk/)", - "LocalcomBot/1.2.x ( http://www.local.com/bot.htm)", - "Lockstep Spider/1.0", - "Look.com", - "Lovel as 1.0 ( +http://www.everatom.com)", - "LTI/LemurProject Nutch Spider/Nutch-1.0-dev (lti crawler for CMU; http://www.lti.cs.cmu.edu; changkuk at cmu dot edu)", - "LTI/LemurProject Nutch Spider/Nutch-1.0-dev (Research spider using Nutch; http://www.lemurproject.org; mhoy@cs.cmu.edu)", - "lwp-trivial/1.32", - "lwp-trivial/1.34", - "lwp-trivial/1.34", - "LWP::Simple/5.22", - "LWP::Simple/5.36", - "LWP::Simple/5.48", - "LWP::Simple/5.50", - "LWP::Simple/5.51", - "LWP::Simple/5.53", - "LWP::Simple/5.63", - "LWP::Simple/5.803", - "Lycos_Spider_(modspider)", - "Lycos_Spider_(T-Rex)", - "Lynx/2.8.4rel.1 libwww-FM/2.14 SSL-MM/1.4.1 OpenSSL/0.9.6c (human-guided@lerly.net)", - "Mac Finder 1.0.xx", - "Mackster( http://www.ukwizz.com )", - "Mahiti.Com/Mahiti Crawler-1.0 (Mahiti.Com; http://mahiti.com ; mahiti.com)", - "Mail.Ru/1.0", - "mailto:webcraft@bea.com", - "mammoth/1.0 ( http://www.sli-systems.com/)", - "MantraAgent", - "MapoftheInternet.com ( http://MapoftheInternet.com)", - "Mariner/5.1b [de] (Win95; I ;Kolibri gncwebbot)", - "Marketwave Hit List", - "Martini", - "MARTINI", - "Marvin v0.3", - "MaSagool/1.0 (MaSagool; http://sagool.jp/; info@sagool.jp)", - "MasterSeek", - "Mata Hari/2.00 ", - "Matrix S.p.A. - FAST Enterprise Crawler 6 (Unknown admin e-mail address)", - "maxomobot/dev-20051201 (maxomo; http://67.102.134.34:4047/MAXOMO/MAXOMObot.html; maxomobot@maxomo.com)", - "MDbot/1.0 (+http://www.megadownload.net/bot.html)", - "MediaCrawler-1.0 (Experimental)", - "Mediapartners-Google/2.1 ( http://www.googlebot.com/bot.html)", - "MediaSearch/0.1", - "MegaSheep v1.0 (www.searchuk.com internet sheep)", - "Megite2.0 (http://www.megite.com)", - "Mercator-1.x", - "Mercator-2.0", - "Mercator-Scrub-1.1", - "Metaeuro Web Crawler/0.2 (MetaEuro Web Search Clustering Engine; http://www.metaeuro.com; crawler at metaeuro dot com)", - "MetaGer-LinkChecker", - "MetagerBot/0.8-dev (MetagerBot; http://metager.de; )", - "MetaGer_PreChecker0.1", - "Metaspinner/0.01 (Metaspinner; http://www.meta-spinner.de/; support@meta-spinner.de/)", - "metatagsdir/0.7 (+http://metatagsdir.com/directory/)", - "MFC Foundation Class Library 4.0", - "MicroBaz", - "Microsoft Small Business Indexer", - "Microsoft URL Control - 6.00.8xxx", - "MicrosoftPrototypeCrawler (How's my crawling? mailto:newbiecrawler@hotmail.com)", - "Missauga Locate 1.0.0", - "Missigua Locator 1.9", - "Missouri College Browse", - "Misterbot-Nutch/0.7.1 (Misterbot-Nutch; http://www.misterbot.fr; admin@misterbot.fr)", - "Miva (AlgoFeedback@miva.com)", - "Mizzu Labs 2.2", - "MJ12bot/vx.x.x (http://majestic12.co.uk/bot.php?+)", - "MJ12bot/vx.x.x (http://www.majestic12.co.uk/projects/dsearch/mj12bot.php)", - "MJBot (SEO assessment)", - "MLBot (www.metadatalabs.com)", - "MnogoSearch/3.2.xx", - "Mo College 1.9", - "moget/x.x (moget@goo.ne.jp)", - "mogimogi/1.0", - "MojeekBot/0.x (archi; http://www.mojeek.com/bot.html)", - "Morris - Mixcat Crawler ( http://mixcat.com)", - "Mouse-House/7.4 (spider_monkey spider info at www.mobrien.com/sm.shtml)", - "mozDex/0.xx-dev (mozDex; http://www.mozdex.com/en/bot.html; spider@mozdex.com)", - "Mozilla (Mozilla@somewhere.com)", - "Mozilla 4.0(compatible; BotSeer/1.0; +http://botseer.ist.psu.edu)", - "Mozilla/2.0 (compatible; Ask Jeeves)", - "Mozilla/2.0 (compatible; Ask Jeeves/Teoma)", - "Mozilla/2.0 (compatible; Ask Jeeves/Teoma; http://about.ask.com/en/docs/about/webmasters.shtml) ", - "Mozilla/2.0 (compatible; Ask Jeeves/Teoma; http://sp.ask.com/docs/about/tech_crawling.html)", - "Mozilla/2.0 (compatible; EZResult -- Internet Search Engine)", - "Mozilla/2.0 (compatible; NEWT ActiveX; Win32)", - "Mozilla/2.0 (compatible; T-H-U-N-D-E-R-S-T-O-N-E)", - "Mozilla/3.0 (compatible; Fluffy the spider; http://www.searchhippo.com/; info@searchhippo.com)", - "Mozilla/3.0 (compatible; Indy Library)", - "Mozilla/3.0 (compatible; MuscatFerret/1.5.4; claude@euroferret.com)", - "Mozilla/3.0 (compatible; MuscatFerret/1.5; olly@muscat.co.uk)", - "Mozilla/3.0 (compatible; MuscatFerret/1.6.x; claude@euroferret.com)", - "Mozilla/3.0 (compatible; scan4mail (advanced version) http://www.peterspages.net/?scan4mail)", - "Mozilla/3.0 (compatible; ScollSpider; http://www.webwobot.com)", - "Mozilla/3.0 (compatible; Webinator-DEV01.home.iprospect.com/2.56)", - "Mozilla/3.0 (compatible; Webinator-indexer.cyberalert.com/2.56)", - "Mozilla/3.0 (INGRID/3.0 MT; webcrawler@NOSPAMexperimental.net; http://aanmelden.ilse.nl/?aanmeld_mode=webhints)", - "Mozilla/3.0 (Slurp.so/Goo; slurp@inktomi.com; http://www.inktomi.com/slurp.html)", - "Mozilla/3.0 (Slurp/cat; slurp@inktomi.com; http://www.inktomi.com/slurp.html)", - "Mozilla/3.0 (Slurp/si; slurp@inktomi.com; http://www.inktomi.com/slurp.html)", - "Mozilla/3.0 (Vagabondo/1.1 MT; webcrawler@NOSPAMwise-guys.nl; http://webagent.wise-guys.nl/)", - "Mozilla/3.0 (Vagabondo/1.x MT; webagent@wise-guys.nl; http://webagent.wise-guys.nl/)", - "Mozilla/3.0 (Vagabondo/2.0 MT; webcrawler@NOSPAMexperimental.net; http://aanmelden.ilse.nl/?aanmeld_mode=webhints)", - "Mozilla/3.0 (Vagabondo/2.0 MT; webcrawler@NOSPAMwise-guys.nl; http://webagent.wise-guys.nl/)", - "Mozilla/3.01 (Compatible; Links2Go Similarity Engine)", - "Mozilla/4.0", - "Mozilla/4.0 (agadine3.0) www.agada.de", - "Mozilla/4.0 (compatible: AstraSpider V.2.1 : astrafind.com)", - "Mozilla/4.0 (compatible; Vagabondo/2.2; webcrawler at wise-guys dot nl; http://webagent.wise-guys.nl/)", - "Mozilla/4.0 (compatible; Vagabondo/4.0Beta; webcrawler at wise-guys dot nl; http://webagent.wise-guys.nl/)", - "Mozilla/4.0 (compatible; Advanced Email Extractor v2.xx)", - "Mozilla/4.0 (compatible; B_L_I_T_Z_B_O_T)", - "Mozilla/4.0 (compatible; ChristCrawler.com ChristCrawler@ChristCENTRAL.com)", - "Mozilla/4.0 (compatible; crawlx, crawler@trd.overture.com)", - "Mozilla/4.0 (compatible; DAUMOA-video; +http://ws.daum.net/aboutkr.html)", - "Mozilla/4.0 (compatible; FastCrawler3 support-fastcrawler3@fast.no)", - "Mozilla/4.0 (compatible; FDSE robot)", - "Mozilla/4.0 (compatible; GPU p2p crawler http://gpu.sourceforge.net/search_engine.php)", - "Mozilla/4.0 (compatible; grub-client-0.2.x; Crawl your stuff with http://grub.org)", - "Mozilla/4.0 (compatible; grub-client-0.3.x; Crawl your own stuff with http://grub.org)", - "Mozilla/4.0 (compatible; grub-client-2.x)", - "Mozilla/4.0 (compatible; Iplexx Spider/1.0 http://www.iplexx.at)", - "Mozilla/4.0 (compatible; MSIE 4.01; Vonna.com b o t)", - "Mozilla/4.0 (compatible; MSIE 4.01; Windows CE; PPC; 240x320; SPV M700; OpVer 19.123.2.733) OrangeBot-Mobile 2008.0 (mobilesearch.support@orange-ftgroup.com)", - "Mozilla/4.0 (compatible; MSIE 4.0; Windows NT; Site Server 3.0 Robot) Indonesia Interactive", - "Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0) (samualt9@bigfoot.com)", - "Mozilla/4.0 (compatible; MSIE 5.0; NetNose-Crawler 2.0; A New Search Experience: http://www.netnose.com)", - "Mozilla/4.0 (compatible; MSIE 5.0; Windows 95) TrueRobot; 1.5", - "Mozilla/4.0 (compatible; MSIE 5.0; Windows 95) VoilaBot BETA 1.2 (http://www.voila.com/)", - "Mozilla/4.0 (compatible; MSIE 5.0; Windows 95) VoilaBot; 1.6", - "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt; DTS Agent", - "Mozilla/4.0 (compatible; MSIE 5.0; www.galaxy.com; www.psychedelix.com)", - "Mozilla/4.0 (compatible; MSIE 5.0; www.galaxy.com; www.psychedelix.com/; http://www.galaxy.com/info/crawler.html)", - "Mozilla/4.0 (compatible; MSIE 5.0; YANDEX)", - "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 4.0; obot)", - "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 4.0; QXW03018)", - "Mozilla/4.0 (compatible; MSIE 6.0 compatible; Asterias Crawler v4; +http://www.singingfish.com/help/spider.html; webmaster@singingfish.com); SpiderThread Revision: 3.10", - "Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.1) Skampy/0.9.x [en]", - "Mozilla/4.0 (compatible; MSIE 6.0; TargetSeek/1.0; +http://www.targetgroups.net/TargetSeek.html)", - "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; ODP entries t_st; http://tuezilla.de/t_st-odp-entries-agent.html)", - "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; ODP links test; http://tuezilla.de/test-odp-links-agent.html)", - "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; ZoomSpider.net bot; .NET CLR 1.1.4322)", - "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; heritrix/1.3.0 http://www.cs.washington.edu/research/networking/websys/)", - "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; QihooBot 1.0 qihoobot@qihoo.net)", - "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT; MS Search 4.0 Robot)", - "Mozilla/4.0 (compatible; MSIE enviable; DAUMOA 2.0; DAUM Web Robot; Daum Communications Corp., Korea; +http://ws.daum.net/aboutkr.html)", - "Mozilla/4.0 (compatible; MSIE is not me; DAUMOA/1.0.1; DAUM Web Robot; Daum Communications Corp., Korea)", - "Mozilla/4.0 (compatible; NaverBot/1.0; http://help.naver.com/delete_main.asp)", - "Mozilla/4.0 (compatible; SpeedySpider; www.entireweb.com)", - "Mozilla/4.0 (compatible; www.galaxy.com)", - "Mozilla/4.0 (compatible; Y!J; for robot study; keyoshid)", - "Mozilla/4.0 (compatible; Yahoo Japan; for robot study; kasugiya)", - "Mozilla/4.0 (JemmaTheTourist;http://www.activtourist.com)", - "Mozilla/4.0 (MobilePhone SCP-5500/US/1.0) NetFront/3.0 MMP/2.0 (compatible; Googlebot/2.1; http://www.google.com/bot.html)", - "Mozilla/4.0 (MobilePhone SCP-5500/US/1.0) NetFront/3.0 MMP/2.0 FAKE (compatible; Googlebot/2.1; http://www.google.com/bot.html)", - "Mozilla/4.0 (Mozilla; http://www.mozilla.org/docs/en/bot.html; master@mozilla.com)", - "Mozilla/4.0 (Sleek Spider/1.2)", - "Mozilla/4.0 compatible FurlBot/Furl Search 2.0 (FurlBot; http://www.furl.net; wn.furlbot@looksmart.net)", - "Mozilla/4.0 compatible ZyBorg/1.0 (wn.zyborg@looksmart.net; http://www.WISEnutbot.com)", - "Mozilla/4.0 compatible ZyBorg/1.0 (ZyBorg@WISEnutbot.com; http://www.WISEnutbot.com)", - "Mozilla/4.0 compatible ZyBorg/1.0 Dead Link Checker (wn.zyborg@looksmart.net; http://www.WISEnutbot.com)", - "Mozilla/4.0 compatible ZyBorg/1.0 for Homepage (ZyBorg@WISEnutbot.com; http://www.WISEnutbot.com)", - "Mozilla/4.0 efp@gmx.net", - "Mozilla/4.0 [en] (Ask Jeeves Corporate Spider)", - "Mozilla/4.0(compatible; Zealbot 1.0)", - "Mozilla/4.04 (compatible; Dulance bot; +http://www.dulance.com/bot.jsp)", - "Mozilla/4.0_(compatible;_MSIE_5.0;_Windows_95)_TrueRobot/1.4 libwww/5.2.8", - "Mozilla/4.0_(compatible;_MSIE_5.0;_Windows_95)_VoilaBot/1.6 libwww/5.3.2", - "Mozilla/4.6 [en] (http://www.cnet.com/)", - "Mozilla/4.7", - "Mozilla/4.7 (compatible; http://eidetica.com/spider)", - "Mozilla/4.7 (compatible; Intelliseek; http://www.intelliseek.com)", - "Mozilla/4.7 (compatible; Whizbang)", - "Mozilla/4.7 (compatible; WhizBang; http://www.whizbang.com/crawler)", - "Mozilla/4.7 [en](BecomeBot@exava.com)", - "Mozilla/4.7 [en](Exabot@exava.com)", - "Mozilla/4.72 [en] (BACS http://www.ba.be)", - "Mozilla/5.0", - "Mozilla/5.0 (+http://www.eurekster.com/mammoth) Mammoth/0.1", - "Mozilla/5.0 (+http://www.sli-systems.com/) Mammoth/0.1", - "Mozilla/5.0 (Clustered-Search-Bot/1.0; support@clush.com; http://www.clush.com/)", - "Mozilla/5.0 (compatible; +http://www.evri.com/evrinid)", - "Mozilla/5.0 (compatible; 008/0.83; http://www.80legs.com/spider.html;) Gecko/2008032620", - "Mozilla/5.0 (compatible; Abonti/0.8 - http://www.abonti.com)", - "Mozilla/5.0 (compatible; aiHitBot/1.0; +http://www.aihit.com/)", - "Mozilla/5.0 (compatible; AnsearchBot/1.x; +http://www.ansearch.com.au/)", - "Mozilla/5.0 (compatible; archive.org_bot/1.10.0 +http://www.loc.gov/minerva/crawl.html)", - "Mozilla/5.0 (compatible; archive.org_bot/1.13.1x http://crawler.archive.org)", - "Mozilla/5.0 (compatible; archive.org_bot/1.5.0-200506132127 http://crawler.archive.org) Hurricane Katrina", - "Mozilla/5.0 (compatible; Ask Jeeves/Teoma; http://about.ask.com/en/docs/about/webmasters.shtml)", - "Mozilla/5.0 (compatible; BecomeBot/1.23; http://www.become.com/webmasters.html)", - "Mozilla/5.0 (compatible; BecomeBot/1.xx; MSIE 6.0 compatible; http://www.become.com/webmasters.html)", - "Mozilla/5.0 (compatible; BecomeBot/2.0beta; http://www.become.com/webmasters.html)", - "Mozilla/5.0 (compatible; BecomeBot/2.x; MSIE 6.0 compatible; http://www.become.com/site_owners.html)", - "Mozilla/5.0 (compatible; BecomeJPBot/2.3; MSIE 6.0 compatible; +http://www.become.co.jp/site_owners.html)", - "Mozilla/5.0 (compatible; BlogRefsBot/0.1; http://www.blogrefs.com/about/bloggers)", - "Mozilla/5.0 (compatible; Bot; +http://pressemitteilung.ws/spamfilter", - "Mozilla/5.0 (compatible; BuzzRankingBot/1.0; +http://www.buzzrankingbot.com/)", - "Mozilla/5.0 (compatible; Charlotte/1.0b; charlotte@betaspider.com)", - "Mozilla/5.0 (compatible; Charlotte/1.0b; http://www.searchme.com/support/)", - "Mozilla/5.0 (compatible; Crawling jpeg; http://www.yama.info.waseda.ac.jp)", - "Mozilla/5.0 (compatible; de/1.13.2 +http://www.de.com)", - "Mozilla/5.0 (compatible; Diffbot/0.1; +http://www.diffbot.com)", - "Mozilla/5.0 (compatible; DNS-Digger-Explorer/1.0; +http://www.dnsdigger.com)", - "Mozilla/5.0 (compatible; DNS-Digger/1.0; +http://www.dnsdigger.com)", - "Mozilla/5.0 (compatible; EARTHCOM.info/2.01; http://www.earthcom.info)", - "Mozilla/5.0 (compatible; EARTHCOM/2.2; +http://enter4u.eu)", - "Mozilla/5.0 (compatible; Exabot Test/3.0; +http://www.exabot.com/go/robot)", - "Mozilla/5.0 (compatible; FatBot 2.0; http://www.thefind.com/main/CrawlerFAQs.fhtml)", - "Mozilla/5.0 (compatible; Galbot/1.0; +http://www.galbot.com/bot.html)", - "mozilla/5.0 (compatible; genevabot http://www.healthdash.com)", - "Mozilla/5.0 (compatible; Googlebot/2.1; http://www.google.com/bot.html)", - "mozilla/5.0 (compatible; heritrix/1.0.4 http://innovationblog.com)", - "Mozilla/5.0 (compatible; heritrix/1.10.2 +http://i.stanford.edu/)", - "Mozilla/5.0 (compatible; heritrix/1.12.1 +http://newstin.com/)", - "Mozilla/5.0 (compatible; heritrix/1.12.1 +http://www.page-store.com)", - "Mozilla/5.0 (compatible; heritrix/1.12.1 +http://www.page-store.com) [email:paul@page-store.com]", - "mozilla/5.0 (compatible; heritrix/1.3.0 http://archive.crawler.org)", - "Mozilla/5.0 (compatible; heritrix/1.4.0 +http://www.chepi.net)", - "Mozilla/5.0 (compatible; heritrix/1.4t http://www.truveo.com/)", - "Mozilla/5.0 (compatible; heritrix/1.5.0 http://www.l3s.de/~kohlschuetter/projects/crawling/)", - "Mozilla/5.0 (compatible; heritrix/1.5.0-200506231921 http://pandora.nla.gov.au/crawl.html)", - "Mozilla/5.0 (compatible; heritrix/1.6.0 http://www.worio.com/)", - "Mozilla/5.0 (compatible; heritrix/1.7.0 +http://www.greaterera.com/)", - "Mozilla/5.0 (compatible; heritrix/1.x.x +http://www.accelobot.com)", - "Mozilla/5.0 (compatible; heritrix/2.0.0-RC1 +http://www.aol.com)", - "Mozilla/5.0 (compatible; Hermit Search. Com; +http://www.hermitsearch.com)", - "Mozilla/5.0 (compatible; HyperixScoop/1.3; +http://www.hyperix.com)", - "Mozilla/5.0 (compatible; IDBot/1.0; +http://www.id-search.org/bot.html)", - "Mozilla/5.0 (compatible; InterseekWeb/3.x)", - "Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.5 (like Gecko) (Exabot-Thumbnails)", - "Mozilla/5.0 (compatible; LemSpider 0.1)", - "Mozilla/5.0 (compatible; MojeekBot/2.0; http://www.mojeek.com/bot.html)", - "Mozilla/5.0 (compatible; MSIE 6.0; Podtech Network; crawler_admin@podtech.net)", - "Mozilla/5.0 (compatible; OnetSzukaj/5.0; http://szukaj.onet.pl)", - "Mozilla/5.0 (compatible; PalmeraBot; http://www.links24h.com/help/palmera) Version 0.001", - "Mozilla/5.0 (compatible; pogodak.ba/3.x)", - "Mozilla/5.0 (compatible; Pogodak.hr/3.1)", - "Mozilla/5.0 (compatible; PWeBot/3.1; http://www.programacionweb.net/robot.php)", - "Mozilla/5.0 (compatible; Quantcastbot/1.0; www.quantcast.com)", - "Mozilla/5.0 (compatible; ScoutJet; +http://www.scoutjet.com/)", - "Mozilla/5.0 (compatible; Scrubby/2.2; http://www.scrubtheweb.com/)", - "Mozilla/5.0 (compatible; ShunixBot/1.x.x +http://www.shunix.com/robot.htm)", - "Mozilla/5.0 (compatible; ShunixBot/1.x; http://www.shunix.com/bot.htm)", - "Mozilla/5.0 (compatible; SkreemRBot +http://skreemr.com)", - "Mozilla/5.0 (compatible; SummizeBot +http://www.summize.com)", - "Mozilla/5.0 (compatible; Synoobot/0.9; http://www.synoo.com/search/bot.html)", - "Mozilla/5.0 (compatible; Theophrastus/x.x; http://users.cs.cf.ac.uk/N.A.Smith/theophrastus.php)", - "Mozilla/5.0 (compatible; TridentSpider/3.1)", - "Mozilla/5.0 (compatible; Vagabondo/2.1; webcrawler at wise-guys dot nl; http://webagent.wise-guys.nl/)", - "Mozilla/5.0 (compatible; Webduniabot/1.0; +http://search.webdunia.com/bot.aspx)", - "Mozilla/5.0 (compatible; worio bot heritrix/1.10.0 +http://worio.com)", - "Mozilla/5.0 (compatible; WoW Lemmings Kathune/2.0;http://www.wowlemmings.com/kathune.html)", - "Mozilla/5.0 (compatible; Yahoo! DE Slurp; http://help.yahoo.com/help/us/ysearch/slurp)", - "Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)", - "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)", - "Mozilla/5.0 (compatible; Yoono; http://www.yoono.com/)", - "Mozilla/5.0 (compatible; YoudaoBot/1.0; http://www.youdao.com/help/webmaster/spider/; )", - "Mozilla/5.0 (compatible; Zenbot/1.3; +http://zen.co.za/webmasters/)", - "Mozilla/5.0 (compatible; zermelo +http://www.powerset.com) [email:paul@page-store.com,crawl@powerset.com]", - "Mozilla/5.0 (compatible;archive.org_bot/1.7.1; collectionId=316; Archive-It; +http://www.archive-it.org)", - "Mozilla/5.0 (compatible;archive.org_bot/heritrix-1.9.0-200608171144 +http://pandora.nla.gov.au/crawl.html)", - "Mozilla/5.0 (compatible;MAINSEEK_BOT)", - "Mozilla/5.0 (Slurp/cat; slurp@inktomi.com; http://www.inktomi.com/slurp.html)", - "Mozilla/5.0 (Slurp/si; slurp@inktomi.com; http://www.inktomi.com/slurp.html)", - "Mozilla/5.0 (Twiceler-0.9 http://www.cuill.com/twiceler/robot.html)", - "Mozilla/5.0 (Version: xxxx Type:xx)", - "Mozilla/5.0 (wgao@genieknows.com)", - "Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.7.7) NimbleCrawler 1.11 obeys UserAgent NimbleCrawler For problems contact: crawler_at_dataalchemy.com", - "Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.8.1) VoilaBot BETA 1.2 (support.voilabot@orange-ftgroup.com)", - "Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.8.1) VoilaBot BETA 1.2 (support.voilabot@orange-ftgroup.com)", - "Mozilla/5.0 (Windows;) NimbleCrawler 1.12 obeys UserAgent NimbleCrawler For problems contact: crawler@health", - "Mozilla/5.0 (Windows;) NimbleCrawler 1.12 obeys UserAgent NimbleCrawler For problems contact: crawler@healthline.com", - "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.2.1; aggregator:Spinn3r (Spinn3r 3.1); http://spinn3r.com/robot) Gecko/20021130", - "Mozilla/5.0 URL-Spider", - "Mozilla/5.0 usww.com-Spider-for-w8.net", - "Mozilla/5.0 wgao@genieknows.com", - "Mozilla/5.0 [en] (compatible; Gulper Web Bot 0.2.4 www.ecsl.cs.sunysb.edu/~maxim/cgi-bin/Link/GulperBot)", - "MQbot metaquerier.cs.uiuc.edu/crawler", - "MQBOT/Nutch-0.9-dev (MQBOT Nutch Crawler; http://falcon.cs.uiuc.edu; mqbot@cs.uiuc.edu)", - "msnbot-media/1.0 (+http://search.msn.com/msnbot.htm)", - "msnbot-Products/1.0 (+http://search.msn.com/msnbot.htm)", - "MSNBOT/0.xx (http://search.msn.com/msnbot.htm)", - "msnbot/x.xx ( http://search.msn.com/msnbot.htm)", - "MSNBOT_Mobile MSMOBOT Mozilla/2.0 (compatible; MSIE 4.02; Windows CE; Default)", - "MSNPTC/1.0", - "MSRBOT (http://research.microsoft.com/research/sv/msrbot)", - "multicrawler ( http://sw.deri.org/2006/04/multicrawler/robots.html)", - "MultiText/0.1", - "MusicWalker2.0 ( http://www.somusical.com)", - "MVAClient", - "Mylinea.com Crawler 2.0", - "Naamah 1.0.1/Blogbot (http://blogbot.de/)", - "Naamah 1.0a/Blogbot (http://blogbot.de/)", - "NABOT/5.0", - "nabot_1.0", - "NameOfAgent (CMS Spider)", - "NASA Search 1.0", - "NationalDirectory-WebSpider/1.3", - "NationalDirectoryAddURL/1.0", - "NaverBot-1.0 (NHN Corp. / +82-2-3011-1954 / nhnbot@naver.com)", - "NaverBot_dloader/1.5", - "NavissoBot", - "NavissoBot/1.7 (+http://navisso.com/)", - "NCSA Beta 1 (http://vias.ncsa.uiuc.edu/viasarchivinginformation.html)", - "Nebullabot/2.2 (http://bot.nebulla.info)", - "NEC Research Agent -- compuman at research.nj.nec.com", - "Net-Seekr Bot/Net-Seekr Bot V1 (http://www.net-seekr.com)", - "NetinfoBot/1.0 (http://netinfo.bg/netinfobot.html)", - "NetLookout/2.24", - "Netluchs/0.8-dev ( ; http://www.netluchs.de/; ___don't___spam_me_@netluchs.de)", - "NetNoseCrawler/v1.0", - "Netprospector JavaCrawler", - "NetResearchServer(http://www.look.com)", - "NetResearchServer/x.x(loopimprovements.com/robot.html)", - "NetSeer/Nutch-0.9 (NetSeer Crawler; http://www.netseer.com; crawler@netseer.com)", - "NetSprint -- 2.0", - "NetWhatCrawler/0.06-dev (NetWhatCrawler from NetWhat.com; http://www.netwhat.com; support@netwhat.com)", - "NetZippy", - "NextGenSearchBot 1 (for information visit http://www.eliyon.com/NextGenSearchBot)", - "NextopiaBOT (+http://www.nextopia.com) distributed crawler client beta v0.x", - "NG-Search/0.90 (NG-SearchBot; http://www.ng-search.com; )", - "NG/1.0", - "NG/4.0.1229", - "NITLE Blog Spider/0.01", - "Noago Spider", - "Nokia-WAPToolkit/1.2 googlebot(at)googlebot.com", - "Nokia6610/1.0 (3.09) Profile/MIDP-1.0 Configuration/CLDC-1.0 (compatible;YahooSeeker/M1A1-R2D2; http://help.yahoo.com/help/us/ysearch/crawling/crawling-01.html)", - "NokodoBot/1.x (+http://nokodo.com/bot.htm)", - "Norbert the Spider(Burf.com)", - "noxtrumbot/1.0 (crawler@noxtrum.com)", - "noyona_0_1", - "NP/0.1 (NP; http://www.nameprotect.com; npbot@nameprotect.com)", - "NPBot (http://www.nameprotect.com/botinfo.html)", - "NPBot-1/2.0", - "Nsauditor/1.x", - "nsyght.com/Nutch-1.0-dev (nsyght.com; Nsyght.com)", - "nsyght.com/Nutch-x.x (nsyght.com; search.nsyght.com)", - "nttdirectory_robot/0.9 (super-robot@super.navi.ocn.ne.jp)", - "nuSearch Spider www.nusearch.com (compatible; MSIE 4.01)", - "NuSearch Spider (compatible; MSIE 6.0)", - "NuSearch Spider www.nusearch.com", - "Nutch", - "Nutch crawler/Nutch-0.9 (picapage.com; admin@picapage.com)", - "Nutch/Nutch-0.9 (Eurobot; http://www.ayell.eu )", - "NutchCVS/0.0x-dev (Nutch; http://www.nutch.org/docs/bot.html; nutch-agent@lists.sourceforge.net)", - "NutchCVS/0.7.1 (Nutch running at UW; http://www.nutch.org/docs/en/bot.html; sycrawl@cs.washington.edu)", - "NutchEC2Test/Nutch-0.9-dev (Testing Nutch on Amazon EC2.; http://lucene.apache.org/nutch/bot.html; ec2test at lucene.com)", - "NutchOrg/0.0x-dev (Nutch; http://www.nutch.org/docs/bot.html; nutch-agent@lists.sourceforge.net)", - "nutchsearch/Nutch-0.9 (Nutch Search 1.0; herceg_novi at yahoo dot com)", - "NutchVinegarCrawl/Nutch-0.8.1 (Vinegar; http://www.cs.washington.edu; eytanadar at gmail dot com)", - "obidos-bot (just looking for books.)", - "ObjectsSearch/0.01-dev (ObjectsSearch;http://www.ObjectsSearch.com/bot.html; support@thesoftwareobjects.com)", - "ObjectsSearch/0.0x (ObjectsSearch; http://www.ObjectsSearch.com/bot.html; support@thesoftwareobjects.com)", - "oBot ((compatible;Win32))", - "Ocelli/1.x (http://www.globalspec.com/Ocelli)", - "Octora Beta - www.octora.com", - "Octora Beta Bot - www.octora.com", - "OmniExplorer_Bot/1.0x (+http://www.omni-explorer.com) Internet CategorizerOmniExplorer http://www.omni-explorer.com/ car & shopping search (64.62.175.xxx)", - "OmniExplorer_Bot/1.0x (+http://www.omni-explorer.com) Job Crawler", - "OmniExplorer_Bot/1.1x (+http://www.omni-explorer.com) Torrent Crawler", - "OmniExplorer_Bot/x.xx (+http://www.omni-explorer.com) WorldIndexer", - "Onet.pl SA- http://szukaj.onet.pl", - "OntoSpider/1.0 libwww-perl/5.65", - "OOZBOT/0.20 ( http://www.setooz.com/oozbot.html ; agentname at setooz dot_com )", - "OpenAcoon v4.0.x (www.openacoon.de)", - "Openbot/3.0+(robot-response@openfind.com.tw;+http://www.openfind.com.tw/robot.html)", - "Openfind data gatherer- Openbot/3.0+(robot-response@openfind.com.tw;+http://www.openfind.com.tw/robot.html)", - "Openfind Robot/1.1A2", - "OpenISearch/1.x (www.openisearch.com)", - "OpenTaggerBot (http://www.opentagger.com/opentaggerbot.htm)", - "OpenTextSiteCrawler/2.9.2", - "OpenWebSpider/0.x.x (http://www.openwebspider.org)", - "OpenWebSpider/x", - "OpidooBOT (larbin2.6.3@unspecified.mail)", - "Oracle Ultra Search", - "OrangeSpider", - "Orbiter/T-2.0 (+http://www.dailyorbit.com/bot.htm)", - "Overture-WebCrawler/3.8/Fresh (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)", - "ozelot/2.7.3 (Search engine indexer; www.flying-cat.de/ozelot; ozelot@flying-cat.de)", - "PADLibrary Spider", - "PageBitesHyperBot/600 (http://www.pagebites.com/)", - "Pagebull http://www.pagebull.com/", - "page_verifier (http://www.securecomputing.com/goto/pv)", - "parallelContextFocusCrawler1.1parallelContextFocusCrawler1.1", - "ParaSite/1.0b (http://www.ianett.com/parasite/)", - "Patwebbot (http://www.herz-power.de/technik.html)", - "PBrowse 1.4b", - "pd02_1.0.0 pd02_1.0.0@dzimi@post.sk", - "PEERbot www.peerbot.com", - "PEval 1.4b", - "PicoSearch/1.0", - "Piffany_Web_Scraper_v0.x", - "Piffany_Web_Spider_v0.x", - "pipeLiner/0.3a (PipeLine Spider;http://www.pipeline-search.com/webmaster.html; webmaster'at'pipeline-search.com)", - "pipeLiner/0.xx (PipeLine Spider; http://www.pipeline-search.com/webmaster.html)", - "Pita", - "PJspider/3.0 (pjspider@portaljuice.com; http://www.portaljuice.com)", - "PlagiarBot/1.0", - "PluckFeedCrawler/2.0 (compatible; Mozilla 4.0; MSIE 5.5; http://www.pluck.com; 1 subscribers)", - "Pluggd/Nutch-0.9 (automated crawler http://www.pluggd.com;support at pluggd dot com)", - "Poirot", - "polybot 1.0 (http://cis.poly.edu/polybot/)", - "Pompos/1.x http://dir.com/pompos.html", - "Pompos/1.x pompos@iliad.fr", - "Popdexter/1.0", - "Port Huron Labs", - "PortalBSpider/2.0 (spider@portalb.com)", - "potbot 1.0", - "PRCrawler/Nutch-0.9 (data mining development project; crawler@projectrialto.com)", - "PrivacyFinder Cache Bot v1.0", - "PrivacyFinder/1.1", - "Production Bot 0116B", - "Production Bot 2016B", - "Production Bot DOT 3016B", - "Program Shareware 1.0.2", - "Project XP5 [2.03.07-111203]", - "PROve AnswerBot 4.0", - "ProWebGuide Link Checker (http://www.prowebguide.com)", - "psbot/0.1 (+http://www.picsearch.com/bot.html)", - "PSurf15a 11", - "PSurf15a 51", - "PSurf15a VA", - "psycheclone", - "PubCrawl (pubcrawl.stanford.edu)", - "pulseBot (pulse Web Miner)", - "PWeBot/1.2 Inspector (http://www.programacionweb.net/robot.php)", - "PycURL", - "Python-urllib/1.1x", - "Python-urllib/2.0a1", - "Qango.com Web Directory (http://www.qango.com/)", - "QEAVis Agent/Nutch-0.9 (Quantitative Evaluation of Academic Websites Visibility; http://nlp.uned.es/qeavis", - "QPCreep Test Rig ( We are not indexing- just testing )", - "QuepasaCreep ( crawler@quepasacorp.com )", - "QuepasaCreep v0.9.1x", - "QueryN Metasearch", - "QweeryBot/3.01 ( http://qweerybot.qweery.nl)", - "Qweery_robot.txt_CheckBot/3.01 (http://qweerybot.qweery.com)", - "R6_CommentReader_(www.radian6.com/crawler)", - "R6_FeedFetcher_(www.radian6.com/crawler)", - "rabaz (rabaz at gigabaz dot com)", - "RaBot/1.0 Agent-admin/phortse@hanmail.net", - "ramBot xtreme x.x", - "RAMPyBot - www.giveRAMP.com/0.1 (RAMPyBot - www.giveRAMP.com; http://www.giveramp.com/bot.html; support@giveRAMP.com)", - "RAMPyBot/0.8-dev (Nutch; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)", - "Rankivabot/3.2 (www.rankiva.com; 3.2; vzmxikn)", - "Rational SiteCheck (Windows NT)", - "Reaper [2.03.10-031204] (http://www.sitesearch.ca/reaper/)", - "Reaper/2.0x (+http://www.sitesearch.ca/reaper)", - "RedCarpet/1.2 (http://www.redcarpet-inc.com/robots.html)", - "RedCell/0.1 (InfoSec Search Bot (Coming Soon); http://www.telegenetic.net/bot.html; lhall@telegenetic.net)", - "RedCell/0.1 (RedCell; telegenetic.net/bot.html; lhall_at_telegenetic.net)", - "RedKernel WWW-Spider 2/0 (+http://www-spider.redkernel-softwares.com/)", - "rico/0.1", - "RixBot (http://babelserver.org/rix)", - "RoboCrawl (http://www.canadiancontent.net)", - "RoboCrawl (www.canadiancontent.net)", - "RoboPal (http://www.findpal.com/)", - "Robot/www.pj-search.com", - "Robot: NutchCrawler- Owner: wdavies@acm.org", - "Robot@SuperSnooper.Com", - "Robozilla/1.0", - "Rotondo/3.1 libwww/5.3.1", - "RRC (crawler_admin@bigfoot.com)", - "RSSMicro.com RSS/Atom Feed Robot", - "RSurf15a 41", - "RSurf15a 51", - "RSurf15a 81", - "RufusBot (Rufus Web Miner; http://64.124.122.252/feedback.html)", - "RufusBot (Rufus Web Miner; http://www.webaroo.com/rooSiteOwners.html)", - "sait/Nutch-0.9 (SAIT Research; http://www.samsung.com)", - "SandCrawler - Compatibility Testing", - "SapphireWebCrawler/1.0 (Sapphire Web Crawler using Nutch; http://boston.lti.cs.cmu.edu/crawler/; mhoy@cs.cmu.edu)", - "SapphireWebCrawler/Nutch-1.0-dev (Sapphire Web Crawler using Nutch; http://boston.lti.cs.cmu.edu/crawler/; mhoy@cs.cmu.edu)", - "savvybot/0.2", - "SBIder/0.7 (SBIder; http://www.sitesell.com/sbider.html; http://support.sitesell.com/contact-support.html)", - "SBIder/0.8-dev (SBIder; http://www.sitesell.com/sbider.html; http://support.sitesell.com/contact-support.html)", - "ScanWeb", - "ScholarUniverse/0.8 (Nutch;+http://scholaruniverse.com/bot.jsp; fetch-agent@scholaruniverse.com)", - "schwarzmann.biz-Spider_for_paddel.org+(http://www.innerprise.net/usp-spider.asp)", - "ScollSpider/2.0 (+http://www.webwobot.com/ScollSpider.php)", - "Scooter-3.0.EU", - "Scooter-3.0.FS", - "Scooter-3.0.HD", - "Scooter-3.0.VNS", - "Scooter-3.0QI", - "Scooter-3.2", - "Scooter-3.2.BT", - "Scooter-3.2.DIL", - "Scooter-3.2.EX", - "Scooter-3.2.JT", - "Scooter-3.2.NIV", - "Scooter-3.2.SF0", - "Scooter-3.2.snippet", - "Scooter-3.3dev", - "Scooter-ARS-1.1", - "Scooter-ARS-1.1-ih", - "scooter-venus-3.0.vns", - "Scooter-W3-1.0", - "Scooter-W3.1.2", - "Scooter/1.0", - "Scooter/1.0 scooter@pa.dec.com", - "Scooter/1.1 (custom)", - "Scooter/2.0 G.R.A.B. V1.1.0", - "Scooter/2.0 G.R.A.B. X2.0", - "Scooter/3.3", - "Scooter/3.3.QA.pczukor", - "Scooter/3.3.vscooter", - "Scooter/3.3_SF", - "Scooter2_Mercator_x-x.0", - "Scooter_bh0-3.0.3", - "Scooter_trk3-3.0.3", - "ScoutAbout", - "ScoutAnt/0.1; +http://www.ant.com/what_is_ant.com/", - "scoutmaster", - "Scrubby/2.x (http://www.scrubtheweb.com/)", - "Scrubby/3.0 (+http://www.scrubtheweb.com/help/technology.html)", - "Search+", - "Search-Engine-Studio", - "search.ch V1.4", - "search.ch V1.4.2 (spiderman@search.ch; http://www.search.ch)", - "Search/1.0 (http://www.innerprise.net/es-spider.asp)", - "searchbot admin@google.com", - "SearchByUsa/2 (SearchByUsa; http://www.SearchByUsa.com/bot.html; info@SearchByUsa.com)", - "SearchdayBot", - "SearchExpress Spider0.99", - "SearchGuild/DMOZ/Experiment (searchguild@gmail.com)", - "SearchGuild_DMOZ_Experiment (chris@searchguild.com)", - "Searchit-Now Robot/2.2 (+http://www.searchit-now.co.uk)", - "Searchmee! Spider v0.98a", - "SearchSight/2.0 (http://SearchSight.com/)", - "SearchSpider.com/1.1", - "Searchspider/1.2 (SearchSpider; http://www.searchspider.com; webmaster@searchspider.com)", - "SearchTone2.0 - IDEARE", - "Seekbot/1.0 (http://www.seekbot.net/bot.html) HTTPFetcher/0.3", - "Seekbot/1.0 (http://www.seekbot.net/bot.html) RobotsTxtFetcher/1.0 (XDF)", - "Seekbot/1.0 (http://www.seekbot.net/bot.html) RobotsTxtFetcher/1.2", - "Seeker.lookseek.com", - "Semager/1.1 (http://www.semager.de/blog/semager-bots/)", - "Semager/1.x (http://www.semager.de)", - "Sensis Web Crawler (search_comments\\at\\sensis\\dot\\com\\dot\\au)", - "Sensis.com.au Web Crawler (search_comments\\at\\sensis\\dot\\com\\dot\\au)", - "SeznamBot/1.0", - "SeznamBot/1.0 (+http://fulltext.seznam.cz/)", - "SeznamBot/2.0-test (+http://fulltext.sblog.cz/)", - "ShablastBot 1.0", - "Shim Crawler", - "Shim-Crawler(Mozilla-compatible; http://www.logos.ic.i.u-tokyo.ac.jp/crawler/; crawl@logos.ic.i.u-tokyo.ac.jp)", - "ShopWiki/1.0 ( +http://www.shopwiki.com/)", - "ShopWiki/1.0 ( +http://www.shopwiki.com/wiki/Help:Bot)", - "Shoula.com Crawler 2.0", - "SietsCrawler/1.1 (+http://www.siets.biz)", - "Sigram/Nutch-1.0-dev (Test agent for Nutch development; http://www.sigram.com/bot.html; bot at sigram dot com)", - "Siigle Orumcex v.001 Turkey (http://www.siigle.com)", - "silk/1.0", - "silk/1.0 (+http://www.slider.com/silk.htm)/3.7", - "Sirketcebot/v.01 (http://www.sirketce.com/bot.html)", - "SiteSpider +(http://www.SiteSpider.com/)", - "SiteTruth.com site rating system", - "SiteXpert", - "Skampy/0.9.x (http://www.skaffe.com/skampy-info.html)", - "Skimpy/0.x (http://www.skaffe.com/skampy-info.html)", - "Skywalker/0.1 (Skywalker; anonymous; anonymous)", - "Slarp/0.1", - "Slider_Search_v1-de", - "Slurp/2.0 (slurp@inktomi.com; http://www.inktomi.com/slurp.html)", - "Slurp/2.0-KiteWeekly (slurp@inktomi.com; http://www.inktomi.com/slurp.html)", - "Slurp/si (slurp@inktomi.com; http://www.inktomi.com/slurp.html)", - "Slurpy Verifier/1.0", - "SlySearch (slysearch@slysearch.com)", - "SlySearch/1.0 http://www.plagiarism.org/crawler/robotinfo.html", - "SlySearch/1.x http://www.slysearch.com", - "smartwit.com", - "SmiffyDCMetaSpider/1.0", - "snap.com beta crawler v0", - "Snapbot/1.0", - "Snapbot/1.0 (Snap Shots, +http://www.snap.com)", - "SnykeBot/0.6 (http://www.snyke.com)", - "SocSciBot ()", - "SoftHypermarketFileCheckBot/1.0+(+http://www.softhypermaket.com)", - "sogou develop spider", - "Sogou Orion spider/3.0(+http://www.sogou.com/docs/help/webmasters.htm#07)", - "sogou spider", - "Sogou web spider/3.0(+http://www.sogou.com/docs/help/webmasters.htm#07)", - "sohu agent", - "sohu-search", - "Sosospider+(+http://help.soso.com/webspider.htm)", - "speedfind ramBot xtreme 8.1", - "Speedy Spider (Beta/x.x; speedy@entireweb.com)", - "Speedy Spider (Entireweb; Beta/1.0; http://www.entireweb.com/about/search_tech/speedyspider/)", - "Speedy_Spider (http://www.entireweb.com)", - "Sphere Scout&v4.0 - scout at sphere dot com", - "Sphider", - "Spida/0.1", - "Spider-Sleek/2.0 (+http://search-info.com/linktous.html)", - "spider.batsch.com", - "spider.yellopet.com - www.yellopet.com", - "Spider/maxbot.com admin@maxbot.com", - "SpiderKU/0.x", - "SpiderMan", - "SpiderMonkey/7.0x (SpiderMonkey.ca info at http://spidermonkey.ca/sm.shtml)", - "Spinne/2.0", - "Spinne/2.0 med", - "Spinne/2.0 med_AH", - "Spock Crawler (http://www.spock.com/crawler)", - "sportsuchmaschine.de-Robot (Version: 1.02- powered by www.sportsuchmaschine.de)", - "sproose/0.1-alpha (sproose crawler; http://www.sproose.com/bot.html; crawler@sproose.com)", - "Sqworm/2.9.81-BETA (beta_release; 20011102-760; i686-pc-linux-gnu)", - "Sqworm/2.9.85-BETA (beta_release; 20011115-775; i686-pc-linux-gnu)", - "SSurf15a 11 ", - "StackRambler/x.x ", - "stat statcrawler@gmail.com", - "Steeler/1.x (http://www.tkl.iis.u-tokyo.ac.jp/~crawler/)", - "Steeler/3.3 (http://www.tkl.iis.u-tokyo.ac.jp/~crawler/)", - "Strategic Board Bot (+http://www.strategicboard.com)", - "Strategic Board Bot (+http://www.strategicboard.com)", - "Submission Spider at surfsafely.com", - "suchbaer.de", - "suchbaer.de (CrawlerAgent v0.103)", - "suchbot", - "Suchknecht.at-Robot", - "suchpadbot/1.0 (+http://www.suchpad.de)", - "SurferF3 1/0", - "suzuran", - "Swooglebot/2.0. (+http://swoogle.umbc.edu/swooglebot.htm)", - "SWSBot-Images/1.2 http://www.smartwaresoft.com/swsbot12.html", - "SygolBot http://www.sygol.net", - "SynoBot", - "Syntryx ANT Scout Chassis Pheromone; Mozilla/4.0 compatible crawler", - "Szukacz/1.x", - "Szukacz/1.x (robot; www.szukacz.pl/jakdzialarobot.html; szukacz@proszynski.pl)", - "tags2dir.com/0.8 (+http://tags2dir.com/directory/)", - "Tagword (http://tagword.com/dmoz_survey.php)", - "Talkro Web-Shot/1.0 (E-mail: webshot@daumsoft.com- Home: http://222.122.15.190/webshot)", - "TCDBOT/Nutch-0.8 (PhD student research;http://www.tcd.ie; mcgettrs at t c d dot IE)", - "TECOMAC-Crawler/0.x", - "Tecomi Bot (http://www.tecomi.com/bot.htm)", - "Teemer (NetSeer, Inc. is a Los Angeles based Internet startup company.; http://www.netseer.com/crawler.html; crawler@netseer.com)", - "Teoma MP", - "teomaagent crawler-admin@teoma.com", - "teomaagent1 [crawler-admin@teoma.com]", - "teoma_agent1", - "Teradex Mapper; mapper@teradex.com; http://www.teradex.com", - "terraminds-bot/1.0 (support@terraminds.de)", - "TerrawizBot/1.0 (+http://www.terrawiz.com/bot.html)", - "Test spider", - "TestCrawler/Nutch-0.9 (Testing Crawler for Research ; http://balihoo.com/index.aspx; tgautier at balihoo dot com)", - "TheRarestParser/0.2a (http://therarestwords.com/)", - "TheSuBot/0.1 (www.thesubot.de)", - "thumbshots-de-Bot (Version: 1.02- powered by www.thumbshots.de)", - "timboBot/0.9 http://www.breakingblogs.com/timbo_bot.html", - "TinEye/1.1 (http://tineye.com/crawler.html)", - "tivraSpider/1.0 (crawler@tivra.com)", - "TJG/Spider", - "Tkensaku/x.x(http://www.tkensaku.com/q.html)", - "Topodia/1.2-dev (Topodia - Crawler for HTTP content indexing; http://www.topodia.com/; support@topodia.com)", - "Toutatis x-xx.x (hoppa.com)", - "Toutatis x.x (hoppa.com)", - "Toutatis x.x-x", - "traazibot/testengine (+http://www.traazi.de)", - "Trampelpfad-Spider", - "Trampelpfad-Spider-v0.1", - "TSurf15a 11", - "Tumblr/1.0 RSS syndication (+http://www.tumblr.com/) (support@tumblr.com)", - "TurnitinBot/x.x (http://www.turnitin.com/robot/crawlerinfo.html)", - "Turnpike Emporium LinkChecker/0.1", - "TutorGig/1.5 (+http://www.tutorgig.com/crawler)", - "Tutorial Crawler 1.4 (http://www.tutorgig.com/crawler)", - "Twiceler www.cuill.com/robots.html", - "Twiceler-0.9 http://www.cuill.com/twiceler/robot.html", - "Tycoon Agent/Nutch-1.0-dev", - "TygoBot", - "TygoProwler", - "UIowaCrawler/1.0", - "UKWizz/Nutch-0.8.1 (UKWizz Nutch crawler; http://www.ukwizz.com/)", - "Ultraseek", - "Under the Rainbow 2.2", - "UofTDB_experiment (leehyun@cs.toronto.edu)", - "updated/0.1-alpha (updated crawler; http://www.updated.com; crawler@updated.com)", - "updated/0.1beta (updated.com; http://www.updated.com; crawler@updated.om)", - "Uptimebot", - "UptimeBot(www.uptimebot.com)", - "URL Spider Pro/x.xx (innerprise.net)", - "urlfan-bot/1.0; +http://www.urlfan.com/site/bot/350.html", - "URL_Spider_Pro/x.x", - "URL_Spider_Pro/x.x+(http://www.innerprise.net/usp-spider.asp)", - "User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)", - "User-Agent: Mozilla/4.0 (SKIZZLE! Distributed Internet Spider v1.0 - www.SKIZZLE.com)", - "USyd-NLP-Spider (http://www.it.usyd.edu.au/~vinci/bot.html)", - "VadixBot", - "Vagabondo-WAP/2.0 (webcrawler at wise-guys dot nl; http://webagent.wise-guys.nl/)/1.0 Profile", - "Vagabondo/1.x MT (webagent@wise-guys.nl)", - "Vagabondo/2.0 MT", - "Vagabondo/2.0 MT (webagent at wise-guys dot nl)", - "Vagabondo/2.0 MT (webagent@NOSPAMwise-guys.nl)", - "Vagabondo/3.0 (webagent at wise-guys dot nl)", - "Vakes/0.01 (Vakes; http://www.vakes.com/; search@vakes.com)", - "versus 0.2 (+http://versus.integis.ch)", - "versus crawler eda.baykan@epfl.ch", - "VeryGoodSearch.com.DaddyLongLegs", - "verzamelgids.nl - Networking4all Bot/x.x", - "Verzamelgids/2.2 (http://www.verzamelgids.nl)", - "Vespa Crawler", - "VisBot/2.0 (Visvo.com Crawler; http://www.visvo.com/bot.html; bot@visvo.com)", - "Vision Research Lab image spider at vision.ece.ucsb.edu", - "VMBot/0.x.x (VMBot; http://www.VerticalMatch.com/; vmbot@tradedot.com)", - "Vortex/2.2 (+http://marty.anstey.ca/robots/vortex/)", - "voyager-hc/1.0", - "voyager/1.0", - "voyager/2.0 (http://www.kosmix.com/html/crawler.html)", - "VSE/1.0 (testcrawler@hotmail.com)", - "VSE/1.0 (testcrawler@vivisimo.com)", - "vspider", - "vspider/3.x", - "VWBOT/Nutch-0.9-dev (VWBOT Nutch Crawler; http://vwbot.cs.uiuc.edu;+vwbot@cs.uiuc.edu", - "W3SiteSearch Crawler_v1.1 http://www.w3sitesearch.de", - "wadaino.jp-crawler 0.2 (http://wadaino.jp/)", - "Wavefire/0.8-dev (Wavefire; http://www.wavefire.com; info@wavefire.com)", - "Waypath development crawler - info at waypath dot com", - "Waypath Scout v2.x - info at waypath dot com", - "Web Snooper", - "web2express.org/Nutch-0.9-dev (leveled playing field; http://web2express.org/; info at web2express.org)", - "WebAlta Crawler/1.2.1 (http://www.webalta.ru/bot.html)", - "WebarooBot (Webaroo Bot; http://64.124.122.252/feedback.html)", - "WebarooBot (Webaroo Bot; http://www.webaroo.com/rooSiteOwners.html)", - "webbandit/4.xx.0", - "Webclipping.com", - "WebCompass 2.0", - "WebCorp/1.0", - "webcrawl.net", - "WebFindBot(http://www.web-find.com)", - "Webglimpse 2.xx.x (http://webglimpse.net)", - "Weblog Attitude Diffusion 1.0", - "webmeasurement-bot, http://rvs.informatik.uni-leipzig.de", - "WebRankSpider/1.37 (+http://ulm191.server4you.de/crawler/)", - "WebSearch.COM.AU/3.0.1 (The Australian Search Engine; http://WebSearch.COM.AU; Search@WebSearch.COM.AU)", - "WebSearchBench WebCrawler v0.1(Experimental)", - "WebsiteWorth v1.0", - "Webspinne/1.0 webmaster@webspinne.de", - "Websquash.com (Add url robot)", - "WebStat/1.0 (Unix; beta; 20040314)", - "Webster v0.3 ( http://webster.healeys.net/ )", - "WebVac (webmaster@pita.stanford.edu)", - "Webverzeichnis.de - Telefon: 01908 / 26005", - "WebVulnCrawl.unknown/1.0 libwww-perl/5.803", - "Wells Search II", - "WEP Search 00", - "WFARC", - "whatUseek_winona/3.0", - "WhizBang! Lab", - "Willow Internet Crawler by Twotrees V2.1", - "WinHTTP Example/1.0", - "WinkBot/0.06 (Wink.com search engine web crawler; http://www.wink.com/Wink:WinkBot; winkbot@wink.com)", - "WIRE/0.11 (Linux; i686; Bot,Robot,Spider,Crawler,aromano@cli.di.unipi.it)", - "WIRE/0.x (Linux; i686; Bot,Robot,Spider,Crawler)", - "WISEbot/1.0 (WISEbot@koreawisenut.com; http://wisebot.koreawisenut.com)", - "worio heritrix bot (+http://worio.com/)", - "woriobot ( http://www.worio.com/)", - "WorldLight", - "Wotbox/alpha0.6 (bot@wotbox.com; http://www.wotbox.com)", - "Wotbox/alpha0.x.x (bot@wotbox.com; http://www.wotbox.com) Java/1.4.1_02", - "WSB WebCrawler V1.0 (Beta)- cl@cs.uni-dortmund.de", - "WSB, http://websearchbench.cs.uni-dortmund.de", - "wume_crawler/1.1 (http://wume.cse.lehigh.edu/~xiq204/crawler/)", - "Wwlib/Linux", - "www.arianna.it", - "WWWeasel Robot v1.00 (http://wwweasel.de)", - "wwwster/1.x (Beta- mailto:gue@cis.uni-muenchen.de)", - "X-Crawler ", - "xirq/0.1-beta (xirq; http://www.xirq.com; xirq@xirq.com)", - "xyro_(xcrawler@cosmos.inria.fr)", - "Y!J-BSC/1.0 (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)", - "Y!J-SRD/1.0", - "Y!J/1.0 (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)", - "yacy (www.yacy.net; v20040602; i386 Linux 2.4.26-gentoo-r13; java 1.4.2_06; MET/en)", - "yacybot (x86 Windows XP 5.1; java 1.5.0_06; Europe/de) yacy.net", - "Yahoo Pipes 1.0", - "Yahoo! Mindset", - "Yahoo-Blogs/v3.9 (compatible; Mozilla 4.0; MSIE 5.5; http://help.yahoo.com/help/us/ysearch/crawling/crawling-02.html )", - "Yahoo-MMAudVid/1.0 (mms dash mmaudvidcrawler dash support at yahoo dash inc dot com)", - "Yahoo-MMAudVid/2.0(mms dash mm aud vid crawler dash support at yahoo dash inc.com ;Mozilla 4.0 compatible; MSIE 7.0;Windows NT 5.0; .NET CLR 2.0)", - "Yahoo-MMCrawler/3.x (mm dash crawler at trd dot overture dot com)", - "Yahoo-Test/4.0", - "Yahoo-VerticalCrawler-FormerWebCrawler/3.9 crawler at trd dot overture dot com; http://www.alltheweb.com/help/webmaster/crawler", - "YahooFeedSeeker/2.0 (compatible; Mozilla 4.0; MSIE 5.5; http://publisher.yahoo.com/rssguide)", - "YahooSeeker-Testing/v3.9 (compatible; Mozilla 4.0; MSIE 5.5; http://search.yahoo.com/)", - "YahooSeeker/1.0 (compatible; Mozilla 4.0; MSIE 5.5; http://help.yahoo.com/help/us/shop/merchant/)", - "YahooSeeker/1.0 (compatible; Mozilla 4.0; MSIE 5.5; http://search.yahoo.com/yahooseeker.html)", - "YahooSeeker/1.1 (compatible; Mozilla 4.0; MSIE 5.5; http://help.yahoo.com/help/us/shop/merchant/)", - "YahooSeeker/bsv3.9 (compatible; Mozilla 4.0; MSIE 5.5; http://help.yahoo.com/help/us/ysearch/crawling/crawling-02.html )", - "YahooSeeker/CafeKelsa-dev (compatible; Konqueror/3.2; FreeBSD ;cafekelsa-dev-webmaster@yahoo-inc.com )", - "Yandex/1.01.001 (compatible; Win16; I)", - "Yanga WorldSearch Bot v1.1/beta (http://www.yanga.co.uk/)", - "yarienavoir.net/0.2", - "Yeti", - "Yeti/0.01 (nhn/1noon, yetibot@naver.com, check robots.txt daily and follows it)", - "Yeti/1.0 (NHN Corp.; http://help.naver.com/robots/)", - "yggdrasil/Nutch-0.9 (yggdrasil biorelated search engine; www dot biotec dot tu minus dresden do de slash schroeder; heiko dot dietze at biotec dot tu minus dresden dot de)", - "YodaoBot/1.0 (http://www.yodao.com/help/webmaster/spider/; )", - "yoofind/yoofind-0.1-dev (yoono webcrawler; http://www.yoono.com ; MyEmail)", - "yoogliFetchAgent/0.1", - "yoono/1.0 web-crawler/1.0", - "YottaCars_Bot/4.12 (+http://www.yottacars.com) Car Search Engine ", - "YottaShopping_Bot/4.12 (+http://www.yottashopping.com) Shopping Search Engine", - "Zao-Crawler", - "Zao-Crawler 0.2b", - "Zao/0.1 (http://www.kototoi.org/zao/)", - "ZBot/1.00 (icaulfield@zeus.com)", - "Zearchit", - "ZeBot_lseek.net (bot@ze.bz)", - "ZeBot_www.ze.bz (ze.bz@hotmail.com)", - "zedzo.digest/0.1 (http://www.zedzo.com/)", - "zermelo Mozilla/5.0 compatible; heritrix/1.12.1 (+http://www.powerset.com) [email:crawl@powerset.com,email:paul@page-store.com]", - "zerxbot/Version 0.6 libwww-perl/5.79", - "Zeus ThemeSite Viewer Webster Pro V2.9 Win32", - "Zeus xxxxx Webster Pro V2.9 Win32", - "Zeusbot/0.07 (Ulysseek's web-crawling robot; http://www.zeusbot.com; agent@zeusbot.com)", - "ZipppBot/0.xx (ZipppBot; http://www.zippp.net; webmaster@zippp.net)", - "ZIPPPCVS/0.xx (ZipppBot/.xx;http://www.zippp.net; webmaster@zippp.net)", - "Zippy v2.0 - Zippyfinder.com", - "ZoomSpider - wrensoft.com", - "zspider/0.9-dev http://feedback.redkolibri.com/", - "ZyBorg/1.0 (ZyBorg@WISEnut.com; http://www.WISEnut.com)"] + file = File.open(File.join(File.dirname(__FILE__), '../../../lib/botlist/') + 'crawler-user-agents.json') + my_hash = JSON.parse(file.read) + list = [] + my_hash.each do |agent| + list << agent['instances'] + end + LIST = list.flatten end end diff --git a/lib/botlist/crawler-user-agents.json b/lib/botlist/crawler-user-agents.json new file mode 100644 index 00000000..7c675e90 --- /dev/null +++ b/lib/botlist/crawler-user-agents.json @@ -0,0 +1,3723 @@ +[ + { + "pattern": "Googlebot\\/", + "url": "http://www.google.com/bot.html", + "instances": [ + "Googlebot/2.1 (+http://www.google.com/bot.html)", + "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", + "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", + "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", + "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", + "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", + "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", + "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Safari/537.36", + "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview Analytics) Chrome/27.0.1453 Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" + ] + } + , + { + "pattern": "Googlebot-Mobile", + "instances": [ + "DoCoMo/2.0 N905i(c100;TB;W24H16) (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)", + "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)", + "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_1 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8B117 Safari/6531.22.7 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)", + "Nokia6820/2.0 (4.83) Profile/MIDP-1.0 Configuration/CLDC-1.0 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)", + "SAMSUNG-SGH-E250/1.0 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Browser/6.2.3.3.c.1.101 (GUI) MMP/2.0 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)" + ] + } + , + { + "pattern": "Googlebot-Image", + "instances": [ + "Googlebot-Image/1.0" + ] + } + , + { + "pattern": "Googlebot-News", + "instances": [ + "Googlebot-News" + ] + } + , + { + "pattern": "Googlebot-Video", + "instances": [ + "Googlebot-Video/1.0" + ] + } + , + { + "pattern": "AdsBot-Google([^-]|$)", + "url": "https://support.google.com/webmasters/answer/1061943?hl=en", + "instances": [ + "AdsBot-Google (+http://www.google.com/adsbot.html)" + ] + } + , + { + "pattern": "AdsBot-Google-Mobile", + "addition_date": "2017/08/21", + "url": "https://support.google.com/adwords/answer/2404197", + "instances": [ + "AdsBot-Google-Mobile-Apps", + "Mozilla/5.0 (Linux; Android 5.0; SM-G920A) AppleWebKit (KHTML, like Gecko) Chrome Mobile Safari (compatible; AdsBot-Google-Mobile; +http://www.google.com/mobile/adsbot.html)", + "Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1 (compatible; AdsBot-Google-Mobile; +http://www.google.com/mobile/adsbot.html)" + ] + } + , + { + "pattern": "Feedfetcher-Google", + "addition_date": "2018/06/27", + "url": "https://support.google.com/webmasters/answer/178852", + "instances": [ + "Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 1 subscribers; feed-id=728742641706423)" + ] + } + , + { + "pattern": "Mediapartners-Google", + "url": "https://support.google.com/webmasters/answer/1061943?hl=en", + "instances": [ + "Mediapartners-Google", + "Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;) Daumoa/4.0 (Following Mediapartners-Google)", + "Mozilla/5.0 (iPhone; U; CPU iPhone OS 10_0 like Mac OS X; en-us) AppleWebKit/602.1.38 (KHTML, like Gecko) Version/10.0 Mobile/14A5297c Safari/602.1 (compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)", + "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_1 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8B117 Safari/6531.22.7 (compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)" + ] + } + , + { + "pattern": "Mediapartners \\(Googlebot\\)", + "addition_date": "2017/08/08", + "url": "https://support.google.com/webmasters/answer/1061943?hl=en", + "instances": [] + } + , + { + "pattern": "APIs-Google", + "addition_date": "2017/08/08", + "url": "https://support.google.com/webmasters/answer/1061943?hl=en", + "instances": [ + "APIs-Google (+https://developers.google.com/webmasters/APIs-Google.html)" + ] + } + , + { + "pattern": "bingbot", + "url": "http://www.bing.com/bingbot.htm", + "instances": [ + "Mozilla/5.0 (Windows Phone 8.1; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 530) like Gecko (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)", + "Mozilla/5.0 (compatible; adidxbot/2.0; http://www.bing.com/bingbot.htm)", + "Mozilla/5.0 (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)", + "Mozilla/5.0 (compatible; bingbot/2.0; http://www.bing.com/bingbot.htm)", + "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm", + "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)", + "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) SitemapProbe", + "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; adidxbot/2.0; http://www.bing.com/bingbot.htm)", + "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)", + "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; http://www.bing.com/bingbot.htm)", + "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)", + "Mozilla/5.0 (seoanalyzer; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)" + ] + } + , + { + "pattern": "Slurp", + "url": "http://help.yahoo.com/help/us/ysearch/slurp", + "instances": [ + "Mozilla/5.0 (compatible; Yahoo! Slurp/3.0; http://help.yahoo.com/help/us/ysearch/slurp)", + "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)", + "Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)" + ] + } + , + { + "pattern": "[wW]get", + "instances": [ + "WGETbot/1.0 (+http://wget.alanreed.org)", + "Wget/1.14 (linux-gnu)" + ] + } + , + { + "pattern": "curl", + "instances": [ + "eCairn-Grabber/1.0 (+http://ecairn.com/grabber) curl/7.15" + ] + } + , + { + "pattern": "LinkedInBot", + "instances": [ + "LinkedInBot/1.0 (compatible; Mozilla/5.0; Jakarta Commons-HttpClient/3.1 +http://www.linkedin.com)", + "LinkedInBot/1.0 (compatible; Mozilla/5.0; Jakarta Commons-HttpClient/4.3 +http://www.linkedin.com)" + ] + } + , + { + "pattern": "Python-urllib", + "instances": [ + "Python-urllib/2.5", + "Python-urllib/2.6", + "Python-urllib/2.7", + "Python-urllib/3.1", + "Python-urllib/3.2", + "Python-urllib/3.3", + "Python-urllib/3.4", + "Python-urllib/3.5", + "Python-urllib/3.6" + ] + } + , + { + "pattern": "python-requests", + "addition_date": "2018/05/27", + "instances": [ + "python-requests/2.18.4" + ] + } + , + { + "pattern": "libwww", + "instances": [ + "2Bone_LinkChecker/1.0 libwww-perl/6.03", + "2Bone_LinkChkr/1.0 libwww-perl/6.03", + "W3C-checklink/2.90 libwww-perl/5.64", + "W3C-checklink/3.6.2.3 libwww-perl/5.64", + "W3C-checklink/4.2 [4.20] libwww-perl/5.803", + "W3C-checklink/4.2.1 [4.21] libwww-perl/5.803", + "W3C-checklink/4.3 [4.42] libwww-perl/5.805", + "W3C-checklink/4.3 [4.42] libwww-perl/5.808", + "W3C-checklink/4.3 [4.42] libwww-perl/5.820", + "W3C-checklink/4.5 [4.154] libwww-perl/5.823", + "W3C-checklink/4.5 [4.160] libwww-perl/5.823", + "amibot - http://www.amidalla.de - tech@amidalla.com libwww-perl/5.831" + ] + } + , + { + "pattern": "httpunit", + "instances": [ + "httpunit/1.x" + ] + } + , + { + "pattern": "nutch", + "instances": [ + "NutchCVS/0.7.1 (Nutch; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)", + "istellabot-nutch/Nutch-1.10" + ] + } + , + { + "pattern": "Go-http-client", + "addition_date": "2016/03/26", + "url": "https://golang.org/pkg/net/http/", + "instances": [ + "Go-http-client/1.1" + ] + } + , + { + "pattern": "phpcrawl", + "addition_date": "2012-09/17", + "url": "http://phpcrawl.cuab.de/", + "instances": [ + "phpcrawl" + ] + } + , + { + "pattern": "msnbot", + "url": "http://search.msn.com/msnbot.htm", + "instances": [ + "adidxbot/1.1 (+http://search.msn.com/msnbot.htm)", + "adidxbot/2.0 (+http://search.msn.com/msnbot.htm)", + "librabot/1.0 (+http://search.msn.com/msnbot.htm)", + "librabot/2.0 (+http://search.msn.com/msnbot.htm)", + "msnbot-NewsBlogs/2.0b (+http://search.msn.com/msnbot.htm)", + "msnbot-UDiscovery/2.0b (+http://search.msn.com/msnbot.htm)", + "msnbot-media/1.0 (+http://search.msn.com/msnbot.htm)", + "msnbot-media/1.1 (+http://search.msn.com/msnbot.htm)", + "msnbot-media/2.0b (+http://search.msn.com/msnbot.htm)", + "msnbot/1.0 (+http://search.msn.com/msnbot.htm)", + "msnbot/1.1 (+http://search.msn.com/msnbot.htm)", + "msnbot/2.0b (+http://search.msn.com/msnbot.htm)", + "msnbot/2.0b (+http://search.msn.com/msnbot.htm).", + "msnbot/2.0b (+http://search.msn.com/msnbot.htm)._" + ] + } + , + { + "pattern": "jyxobot", + "instances": [] + } + , + { + "pattern": "FAST-WebCrawler", + "instances": [ + "FAST-WebCrawler/3.6/FirstPage (atw-crawler at fast dot no;http://fast.no/support/crawler.asp)", + "FAST-WebCrawler/3.7 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)", + "FAST-WebCrawler/3.7/FirstPage (atw-crawler at fast dot no;http://fast.no/support/crawler.asp)", + "FAST-WebCrawler/3.8" + ] + } + , + { + "pattern": "FAST Enterprise Crawler", + "instances": [ + "FAST Enterprise Crawler 6 / Scirus scirus-crawler@fast.no; http://www.scirus.com/srsapp/contactus/", + "FAST Enterprise Crawler 6 used by Schibsted (webcrawl@schibstedsok.no)" + ] + } + , + { + "pattern": "BIGLOTRON", + "instances": [ + "BIGLOTRON (Beta 2;GNU/Linux)" + ] + } + , + { + "pattern": "Teoma", + "instances": [ + "Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://sp.ask.com/docs/about/tech_crawling.html)", + "Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://about.ask.com/en/docs/about/webmasters.shtml)" + ], + "url": "http://about.ask.com/en/docs/about/webmasters.shtml" + } + , + { + "pattern": "convera", + "instances": [ + "ConveraCrawler/0.9e (+http://ews.converasearch.com/crawl.htm)" + ], + "url": "http://ews.converasearch.com/crawl.htm" + } + , + { + "pattern": "seekbot", + "instances": [ + "Seekbot/1.0 (http://www.seekbot.net/bot.html) RobotsTxtFetcher/1.2" + ], + "url": "http://www.seekbot.net/bot.html" + } + , + { + "pattern": "Gigabot", + "instances": [ + "Gigabot/1.0", + "Gigabot/2.0 (http://www.gigablast.com/spider.html)" + ], + "url": "http://www.gigablast.com/spider.html" + } + , + { + "pattern": "Gigablast", + "instances": [ + "GigablastOpenSource/1.0" + ], + "url": "https://github.com/gigablast/open-source-search-engine" + } + , + { + "pattern": "exabot", + "instances": [ + "Mozilla/5.0 (compatible; Alexabot/1.0; +http://www.alexa.com/help/certifyscan; certifyscan@alexa.com)", + "Mozilla/5.0 (compatible; Exabot PyExalead/3.0; +http://www.exabot.com/go/robot)", + "Mozilla/5.0 (compatible; Exabot-Images/3.0; +http://www.exabot.com/go/robot)", + "Mozilla/5.0 (compatible; Exabot/3.0 (BiggerBetter); +http://www.exabot.com/go/robot)", + "Mozilla/5.0 (compatible; Exabot/3.0; +http://www.exabot.com/go/robot)", + "Mozilla/5.0 (compatible; Exabot/3.0; http://www.exabot.com/go/robot)" + ] + } + , + { + "pattern": "ia_archiver", + "instances": [ + "ia_archiver (+http://www.alexa.com/site/help/webmasters; crawler@alexa.com)", + "ia_archiver-web.archive.org" + ] + } + , + { + "pattern": "GingerCrawler", + "instances": [ + "GingerCrawler/1.0 (Language Assistant for Dyslexics; www.gingersoftware.com/crawler_agent.htm; support at ginger software dot com)" + ] + } + , + { + "pattern": "webmon ", + "instances": [] + } + , + { + "pattern": "HTTrack", + "instances": [ + "Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98)" + ] + } + , + { + "pattern": "grub.org", + "instances": [ + "Mozilla/4.0 (compatible; grub-client-0.3.0; Crawl your own stuff with http://grub.org)", + "Mozilla/4.0 (compatible; grub-client-1.0.4; Crawl your own stuff with http://grub.org)", + "Mozilla/4.0 (compatible; grub-client-1.0.5; Crawl your own stuff with http://grub.org)", + "Mozilla/4.0 (compatible; grub-client-1.0.6; Crawl your own stuff with http://grub.org)", + "Mozilla/4.0 (compatible; grub-client-1.0.7; Crawl your own stuff with http://grub.org)", + "Mozilla/4.0 (compatible; grub-client-1.1.1; Crawl your own stuff with http://grub.org)", + "Mozilla/4.0 (compatible; grub-client-1.2.1; Crawl your own stuff with http://grub.org)", + "Mozilla/4.0 (compatible; grub-client-1.3.1; Crawl your own stuff with http://grub.org)", + "Mozilla/4.0 (compatible; grub-client-1.3.7; Crawl your own stuff with http://grub.org)", + "Mozilla/4.0 (compatible; grub-client-1.4.3; Crawl your own stuff with http://grub.org)", + "Mozilla/4.0 (compatible; grub-client-1.5.3; Crawl your own stuff with http://grub.org)" + ] + } + , + { + "pattern": "UsineNouvelleCrawler", + "instances": [] + } + , + { + "pattern": "antibot", + "instances": [] + } + , + { + "pattern": "netresearchserver", + "instances": [] + } + , + { + "pattern": "speedy", + "instances": [ + "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) Speedy Spider (http://www.entireweb.com/about/search_tech/speedy_spider/)", + "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) Speedy Spider for SpeedyAds (http://www.entireweb.com/about/search_tech/speedy_spider/)", + "Mozilla/5.0 (compatible; Speedy Spider; http://www.entireweb.com/about/search_tech/speedy_spider/)", + "Speedy Spider (Entireweb; Beta/1.2; http://www.entireweb.com/about/search_tech/speedyspider/)", + "Speedy Spider (http://www.entireweb.com/about/search_tech/speedy_spider/)" + ] + } + , + { + "pattern": "fluffy", + "instances": [] + } + , + { + "pattern": "bibnum.bnf", + "instances": [ + "Mozilla/5.0 (compatible; bnf.fr_bot; +http://bibnum.bnf.fr/robot/bnf.html)" + ] + } + , + { + "pattern": "findlink", + "instances": [ + "findlinks/1.0 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/1.1.3-beta8 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/1.1.3-beta9 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/1.1.5-beta7 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/1.1.6-beta1 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/1.1.6-beta1 (+http://wortschatz.uni-leipzig.de/findlinks/; YaCy 0.1; yacy.net)", + "findlinks/1.1.6-beta2 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/1.1.6-beta3 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/1.1.6-beta4 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/1.1.6-beta5 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/1.1.6-beta6 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/2.0 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/2.0.1 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/2.0.2 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/2.0.4 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/2.0.5 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/2.0.9 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/2.1 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/2.1.3 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/2.1.5 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/2.2 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/2.5 (+http://wortschatz.uni-leipzig.de/findlinks/)", + "findlinks/2.6 (+http://wortschatz.uni-leipzig.de/findlinks/)" + ] + } + , + { + "pattern": "msrbot", + "instances": [] + } + , + { + "pattern": "panscient", + "instances": [ + "panscient.com" + ] + } + , + { + "pattern": "yacybot", + "instances": [ + "yacybot (/global; amd64 FreeBSD 10.3-RELEASE; java 1.8.0_77; GMT/en) http://yacy.net/bot.html", + "yacybot (/global; amd64 FreeBSD 10.3-RELEASE-p7; java 1.7.0_95; GMT/en) http://yacy.net/bot.html", + "yacybot (-global; amd64 FreeBSD 9.2-RELEASE-p10; java 1.7.0_65; Europe/en) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 2.6.32-042stab093.4; java 1.7.0_65; Etc/en) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 2.6.32-042stab094.8; java 1.7.0_79; America/en) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 2.6.32-042stab108.8; java 1.7.0_91; America/en) http://yacy.net/bot.html", + "yacybot (-global; amd64 Linux 2.6.32-042stab111.11; java 1.7.0_79; Europe/en) http://yacy.net/bot.html", + "yacybot (-global; amd64 Linux 2.6.32-042stab116.1; java 1.7.0_79; Europe/en) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 2.6.32-573.3.1.el6.x86_64; java 1.7.0_85; Europe/en) http://yacy.net/bot.html", + "yacybot (-global; amd64 Linux 3.10.0-229.4.2.el7.x86_64; java 1.7.0_79; Europe/en) http://yacy.net/bot.html", + "yacybot (-global; amd64 Linux 3.10.0-229.4.2.el7.x86_64; java 1.8.0_45; Europe/en) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.10.0-229.7.2.el7.x86_64; java 1.8.0_45; Europe/en) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.10.0-327.22.2.el7.x86_64; java 1.7.0_101; Etc/en) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.11.10-21-desktop; java 1.7.0_51; America/en) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.12.1; java 1.7.0_65; Europe/en) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.13.0-042stab093.4; java 1.7.0_79; Europe/de) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.13.0-042stab093.4; java 1.7.0_79; Europe/en) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.13.0-45-generic; java 1.7.0_75; Europe/en) http://yacy.net/bot.html", + "yacybot (-global; amd64 Linux 3.13.0-61-generic; java 1.7.0_79; Europe/en) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.13.0-74-generic; java 1.7.0_91; Europe/en) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.13.0-83-generic; java 1.7.0_95; Europe/de) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.13.0-83-generic; java 1.7.0_95; Europe/en) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.13.0-85-generic; java 1.7.0_101; Europe/en) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.13.0-85-generic; java 1.7.0_95; Europe/en) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.13.0-88-generic; java 1.7.0_101; Europe/en) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.14-0.bpo.1-amd64; java 1.7.0_55; Europe/de) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.14.32-xxxx-grs-ipv6-64; java 1.7.0_75; Europe/en) http://yacy.net/bot.html", + "yacybot (-global; amd64 Linux 3.14.32-xxxx-grs-ipv6-64; java 1.8.0_111; Europe/de) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_111; Europe/de) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_75; America/en) http://yacy.net/bot.html", + "yacybot (-global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_75; Europe/en) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_75; Europe/en) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_79; Europe/de) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_79; Europe/en) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_91; Europe/de) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_95; Europe/en) http://yacy.net/bot.html", + "yacybot (/global; amd64 Linux 3.16-0.bpo.2-amd64; java 1.7.0_65; Europe/en) http://yacy.net/bot.html", + "yacybot (-global; amd64 Linux 3.19.0-15-generic; java 1.8.0_45-internal; Europe/de) http://yacy.net/bot.html", + "yacybot (-global; amd64 Linux 3.2.0-4-amd64; java 1.7.0_65; Europe/en) http://yacy.net/bot.html", + "yacybot (-global; amd64 Linux 3.2.0-4-amd64; java 1.7.0_67; Europe/en) http://yacy.net/bot.html", + "yacybot (-global; amd64 Linux 4.4.0-57-generic; java 9-internal; Europe/en) http://yacy.net/bot.html", + "yacybot (-global; amd64 Windows 8.1 6.3; java 1.7.0_55; Europe/de) http://yacy.net/bot.html", + "yacybot (-global; amd64 Windows 8 6.2; java 1.7.0_55; Europe/de) http://yacy.net/bot.html" + ] + } + , + { + "pattern": "AISearchBot", + "instances": [] + } + , + { + "pattern": "ips-agent", + "instances": [ + "BlackBerry9000/4.6.0.167 Profile/MIDP-2.0 Configuration/CLDC-1.1 VendorID/102 ips-agent", + "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.12; ips-agent) Gecko/20050922 Fedora/1.0.7-1.1.fc4 Firefox/1.0.7", + "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.1.3; ips-agent) Gecko/20090824 Fedora/1.0.7-1.1.fc4 Firefox/3.5.3", + "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.24; ips-agent) Gecko/20111107 Ubuntu/10.04 (lucid) Firefox/3.6.24", + "Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:14.0; ips-agent) Gecko/20100101 Firefox/14.0.1" + ] + } + , + { + "pattern": "tagoobot", + "instances": [] + } + , + { + "pattern": "MJ12bot", + "instances": [ + "MJ12bot/v1.2.0 (http://majestic12.co.uk/bot.php?+)", + "Mozilla/5.0 (compatible; MJ12bot/v1.2.1; http://www.majestic12.co.uk/bot.php?+)", + "Mozilla/5.0 (compatible; MJ12bot/v1.2.3; http://www.majestic12.co.uk/bot.php?+)", + "Mozilla/5.0 (compatible; MJ12bot/v1.2.4; http://www.majestic12.co.uk/bot.php?+)", + "Mozilla/5.0 (compatible; MJ12bot/v1.2.5; http://www.majestic12.co.uk/bot.php?+)", + "Mozilla/5.0 (compatible; MJ12bot/v1.3.0; http://www.majestic12.co.uk/bot.php?+)", + "Mozilla/5.0 (compatible; MJ12bot/v1.3.1; http://www.majestic12.co.uk/bot.php?+)", + "Mozilla/5.0 (compatible; MJ12bot/v1.3.2; http://www.majestic12.co.uk/bot.php?+)", + "Mozilla/5.0 (compatible; MJ12bot/v1.3.3; http://www.majestic12.co.uk/bot.php?+)", + "Mozilla/5.0 (compatible; MJ12bot/v1.4.0; http://www.majestic12.co.uk/bot.php?+)", + "Mozilla/5.0 (compatible; MJ12bot/v1.4.1; http://www.majestic12.co.uk/bot.php?+)", + "Mozilla/5.0 (compatible; MJ12bot/v1.4.2; http://www.majestic12.co.uk/bot.php?+)", + "Mozilla/5.0 (compatible; MJ12bot/v1.4.3; http://www.majestic12.co.uk/bot.php?+)", + "Mozilla/5.0 (compatible; MJ12bot/v1.4.4 (domain ownership verifier); http://www.majestic12.co.uk/bot.php?+)", + "Mozilla/5.0 (compatible; MJ12bot/v1.4.4; http://www.majestic12.co.uk/bot.php?+)", + "Mozilla/5.0 (compatible; MJ12bot/v1.4.5; http://www.majestic12.co.uk/bot.php?+)", + "Mozilla/5.0 (compatible; MJ12bot/v1.4.6; http://mj12bot.com/)", + "Mozilla/5.0 (compatible; MJ12bot/v1.4.7; http://mj12bot.com/)", + "Mozilla/5.0 (compatible; MJ12bot/v1.4.7; http://www.majestic12.co.uk/bot.php?+)", + "Mozilla/5.0 (compatible; MJ12bot/v1.4.8; http://mj12bot.com/)" + ] + } + , + { + "pattern": "woriobot", + "instances": [ + "Mozilla/5.0 (compatible; woriobot +http://worio.com)", + "Mozilla/5.0 (compatible; woriobot support [at] zite [dot] com +http://zite.com)" + ] + } + , + { + "pattern": "yanga", + "instances": [ + "Yanga WorldSearch Bot v1.1/beta (http://www.yanga.co.uk/)" + ] + } + , + { + "pattern": "buzzbot", + "instances": [ + "Buzzbot/1.0 (Buzzbot; http://www.buzzstream.com; buzzbot@buzzstream.com)" + ] + } + , + { + "pattern": "mlbot", + "instances": [ + "MLBot (www.metadatalabs.com/mlbot)" + ] + } + , + { + "pattern": "YandexBot", + "url": "http://yandex.com/bots", + "instances": [ + "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)" + ], + "addition_date": "2015/04/14" + } + , + { + "pattern": "yandex.com\\/bots", + "url": "https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.xml#robot-in-logs", + "instances": [ + "Mozilla/5.0 (compatible; YandexWebmaster/2.0; +http://yandex.com/bots)", + "Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B411 Safari/600.1.4 (compatible; YandexMobileBot/3.0; +http://yandex.com/bots)" + ], + "addition_date": "2016/12/01" + } + , + { + "pattern": "purebot", + "addition_date": "2010/01/19", + "instances": [] + } + , + { + "pattern": "Linguee Bot", + "addition_date": "2010/01/26", + "url": "http://www.linguee.com/bot", + "instances": [ + "Linguee Bot (http://www.linguee.com/bot)", + "Linguee Bot (http://www.linguee.com/bot; bot@linguee.com)" + ] + } + , + { + "pattern": "CyberPatrol", + "addition_date": "2010/02/11", + "url": "http://www.cyberpatrol.com/cyberpatrolcrawler.asp", + "instances": [ + "CyberPatrol SiteCat Webbot (http://www.cyberpatrol.com/cyberpatrolcrawler.asp)" + ] + } + , + { + "pattern": "voilabot", + "addition_date": "2010/05/18", + "instances": [ + "Mozilla/5.0 (Windows NT 5.1; U; Win64; fr; rv:1.8.1) VoilaBot BETA 1.2 (support.voilabot@orange-ftgroup.com)", + "Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.8.1) VoilaBot BETA 1.2 (support.voilabot@orange-ftgroup.com)", + "Mozilla/5.0 (compatible; OrangeBot/2.0; support.voilabot@orange.com)" + ] + } + , + { + "pattern": "Baiduspider", + "addition_date": "2010/07/15", + "url": "http://www.baidu.jp/spider/", + "instances": [ + "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)" + ] + } + , + { + "pattern": "citeseerxbot", + "addition_date": "2010/07/17", + "instances": [] + } + , + { + "pattern": "spbot", + "addition_date": "2010/07/31", + "url": "http://www.seoprofiler.com/bot", + "instances": [ + "Mozilla/5.0 (compatible; spbot/1.0; +http://www.seoprofiler.com/bot/ )", + "Mozilla/5.0 (compatible; spbot/1.1; +http://www.seoprofiler.com/bot/ )", + "Mozilla/5.0 (compatible; spbot/1.2; +http://www.seoprofiler.com/bot/ )", + "Mozilla/5.0 (compatible; spbot/2.0.1; +http://www.seoprofiler.com/bot/ )", + "Mozilla/5.0 (compatible; spbot/2.0.2; +http://www.seoprofiler.com/bot/ )", + "Mozilla/5.0 (compatible; spbot/2.0.3; +http://www.seoprofiler.com/bot/ )", + "Mozilla/5.0 (compatible; spbot/2.0.4; +http://www.seoprofiler.com/bot )", + "Mozilla/5.0 (compatible; spbot/2.0; +http://www.seoprofiler.com/bot/ )", + "Mozilla/5.0 (compatible; spbot/2.1; +http://www.seoprofiler.com/bot )", + "Mozilla/5.0 (compatible; spbot/3.0; +http://www.seoprofiler.com/bot )", + "Mozilla/5.0 (compatible; spbot/3.1; +http://www.seoprofiler.com/bot )", + "Mozilla/5.0 (compatible; spbot/4.0.1; +http://www.seoprofiler.com/bot )", + "Mozilla/5.0 (compatible; spbot/4.0.2; +http://www.seoprofiler.com/bot )", + "Mozilla/5.0 (compatible; spbot/4.0.3; +http://www.seoprofiler.com/bot )", + "Mozilla/5.0 (compatible; spbot/4.0.4; +http://www.seoprofiler.com/bot )", + "Mozilla/5.0 (compatible; spbot/4.0.5; +http://www.seoprofiler.com/bot )", + "Mozilla/5.0 (compatible; spbot/4.0.6; +http://www.seoprofiler.com/bot )", + "Mozilla/5.0 (compatible; spbot/4.0.7; +http://OpenLinkProfiler.org/bot )", + "Mozilla/5.0 (compatible; spbot/4.0.7; +https://www.seoprofiler.com/bot )", + "Mozilla/5.0 (compatible; spbot/4.0.8; +http://OpenLinkProfiler.org/bot )", + "Mozilla/5.0 (compatible; spbot/4.0.9; +http://OpenLinkProfiler.org/bot )", + "Mozilla/5.0 (compatible; spbot/4.0; +http://www.seoprofiler.com/bot )", + "Mozilla/5.0 (compatible; spbot/4.0a; +http://www.seoprofiler.com/bot )", + "Mozilla/5.0 (compatible; spbot/4.0b; +http://www.seoprofiler.com/bot )", + "Mozilla/5.0 (compatible; spbot/4.1.0; +http://OpenLinkProfiler.org/bot )", + "Mozilla/5.0 (compatible; spbot/4.2.0; +http://OpenLinkProfiler.org/bot )", + "Mozilla/5.0 (compatible; spbot/4.3.0; +http://OpenLinkProfiler.org/bot )", + "Mozilla/5.0 (compatible; spbot/4.4.0; +http://OpenLinkProfiler.org/bot )", + "Mozilla/5.0 (compatible; spbot/4.4.1; +http://OpenLinkProfiler.org/bot )", + "Mozilla/5.0 (compatible; spbot/4.4.2; +http://OpenLinkProfiler.org/bot )", + "Mozilla/5.0 (compatible; spbot/5.0.1; +http://OpenLinkProfiler.org/bot )", + "Mozilla/5.0 (compatible; spbot/5.0.2; +http://OpenLinkProfiler.org/bot )", + "Mozilla/5.0 (compatible; spbot/5.0.3; +http://OpenLinkProfiler.org/bot )", + "Mozilla/5.0 (compatible; spbot/5.0; +http://OpenLinkProfiler.org/bot )" + ] + } + , + { + "pattern": "twengabot", + "addition_date": "2010/08/03", + "url": "http://www.twenga.com/bot.html", + "instances": [] + } + , + { + "pattern": "postrank", + "addition_date": "2010/08/03", + "url": "http://www.postrank.com", + "instances": [ + "PostRank/2.0 (postrank.com)", + "PostRank/2.0 (postrank.com; 1 subscribers)" + ] + } + , + { + "pattern": "turnitinbot", + "addition_date": "2010/09/26", + "url": "http://www.turnitin.com", + "instances": [] + } + , + { + "pattern": "scribdbot", + "addition_date": "2010/09/28", + "url": "http://www.scribd.com", + "instances": [] + } + , + { + "pattern": "page2rss", + "addition_date": "2010/10/07", + "url": "http://www.page2rss.com", + "instances": [ + "Mozilla/5.0 (compatible; Page2RSS/0.7; +http://page2rss.com/)" + ] + } + , + { + "pattern": "sitebot", + "addition_date": "2010/12/15", + "url": "http://www.sitebot.org", + "instances": [ + "Mozilla/5.0 (compatible; Whoiswebsitebot/0.1; +http://www.whoiswebsite.net)" + ] + } + , + { + "pattern": "linkdex", + "addition_date": "2011/01/06", + "url": "http://www.linkdex.com", + "instances": [ + "Mozilla/5.0 (compatible; linkdexbot/2.0; +http://www.linkdex.com/about/bots/)", + "Mozilla/5.0 (compatible; linkdexbot/2.0; +http://www.linkdex.com/bots/)", + "Mozilla/5.0 (compatible; linkdexbot/2.1; +http://www.linkdex.com/about/bots/)", + "Mozilla/5.0 (compatible; linkdexbot/2.1; +http://www.linkdex.com/bots/)", + "Mozilla/5.0 (compatible; linkdexbot/2.2; +http://www.linkdex.com/bots/)", + "linkdex.com/v2.0", + "linkdexbot/Nutch-1.0-dev (http://www.linkdex.com/; crawl at linkdex dot com)" + ] + } + , + { + "pattern": "Adidxbot", + "url": "http://onlinehelp.microsoft.com/en-us/bing/hh204496.aspx", + "instances": [] + } + , + { + "pattern": "blekkobot", + "url": "http://blekko.com/about/blekkobot", + "instances": [ + "Mozilla/5.0 (compatible; Blekkobot; ScoutJet; +http://blekko.com/about/blekkobot)" + ] + } + , + { + "pattern": "ezooms", + "addition_date": "2011/04/27", + "url": "http://www.phpbb.com/community/viewtopic.php?f=64&t=935605&start=450#p12948289", + "instances": [ + "Mozilla/5.0 (compatible; Ezooms/1.0; ezooms.bot@gmail.com)" + ] + } + , + { + "pattern": "dotbot", + "addition_date": "2011/04/27", + "instances": [ + "Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, help@moz.com)", + "dotbot" + ] + } + , + { + "pattern": "Mail.RU_Bot", + "addition_date": "2011/04/27", + "instances": [ + "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/", + "Mozilla/5.0 (compatible; Mail.RU_Bot/2.0; +http://go.mail.ru/" + ] + } + , + { + "pattern": "discobot", + "addition_date": "2011/05/03", + "url": "http://discoveryengine.com/discobot.html", + "instances": [ + "Mozilla/5.0 (compatible; discobot/1.0; +http://discoveryengine.com/discobot.html)", + "Mozilla/5.0 (compatible; discobot/2.0; +http://discoveryengine.com/discobot.html)", + "mozilla/5.0 (compatible; discobot/1.1; +http://discoveryengine.com/discobot.html)" + ] + } + , + { + "pattern": "heritrix", + "addition_date": "2011/06/21", + "url": "http://crawler.archive.org/", + "instances": [ + "Mozilla/5.0 (compatible; archive.org_bot/heritrix-1.15.4 +http://www.archive.org)", + "Mozilla/5.0 (compatible; heritrix/1.12.1 +http://www.webarchiv.cz)", + "Mozilla/5.0 (compatible; heritrix/1.12.1b +http://netarkivet.dk/website/info.html)", + "Mozilla/5.0 (compatible; heritrix/1.14.2 +http://rjpower.org)", + "Mozilla/5.0 (compatible; heritrix/1.14.2 +http://www.webarchiv.cz)", + "Mozilla/5.0 (compatible; heritrix/1.14.3 +http://archive.org)", + "Mozilla/5.0 (compatible; heritrix/1.14.3 +http://www.accelobot.com)", + "Mozilla/5.0 (compatible; heritrix/1.14.3 +http://www.webarchiv.cz)", + "Mozilla/5.0 (compatible; heritrix/1.14.3.r6601 +http://www.buddybuzz.net/yptrino)", + "Mozilla/5.0 (compatible; heritrix/1.14.4 +http://parsijoo.ir)", + "Mozilla/5.0 (compatible; heritrix/1.14.4 +http://www.exif-search.com)", + "Mozilla/5.0 (compatible; heritrix/2.0.2 +http://aihit.com)", + "Mozilla/5.0 (compatible; heritrix/2.0.2 +http://seekda.com)", + "Mozilla/5.0 (compatible; heritrix/3.0.0-SNAPSHOT-20091120.021634 +http://crawler.archive.org)", + "Mozilla/5.0 (compatible; heritrix/3.1.0-RC1 +http://boston.lti.cs.cmu.edu/crawler_12/)", + "Mozilla/5.0 (compatible; heritrix/3.1.1 +http://places.tomtom.com/crawlerinfo)", + "Mozilla/5.0 (compatible; heritrix/3.1.1 +http://www.mixdata.com)", + "Mozilla/5.0 (compatible; heritrix/3.1.1-SNAPSHOT-20120116.200628 +http://www.archive.org/details/archive.org_bot)", + "Mozilla/5.0 (compatible; heritrix/3.1.1; UniLeipzigASV +http://corpora.informatik.uni-leipzig.de/crawler_faq.html)", + "Mozilla/5.0 (compatible; heritrix/3.2.0 +http://www.crim.ca)", + "Mozilla/5.0 (compatible; heritrix/3.2.0 +http://www.exif-search.com)", + "Mozilla/5.0 (compatible; heritrix/3.2.0 +http://www.mixdata.com)", + "Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-20140702-2247 +http://archive.org/details/archive.org_bot)", + "Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-20160309-0050; UniLeipzigASV +http://corpora.informatik.uni-leipzig.de/crawler_faq.html)", + "Mozilla/5.0 (compatible; sukibot_heritrix/3.1.1 +http://suki.ling.helsinki.fi/eng/webmasters.html)" + ] + } + , + { + "pattern": "findthatfile", + "addition_date": "2011/06/21", + "url": "http://www.findthatfile.com/", + "instances": [] + } + , + { + "pattern": "europarchive.org", + "addition_date": "2011/06/21", + "url": "", + "instances": [ + "Mozilla/5.0 (compatible; MSIE 7.0 +http://www.europarchive.org)" + ] + } + , + { + "pattern": "NerdByNature.Bot", + "addition_date": "2011/07/12", + "url": "http://www.nerdbynature.net/bot", + "instances": [ + "Mozilla/5.0 (compatible; NerdByNature.Bot; http://www.nerdbynature.net/bot)" + ] + } + , + { + "pattern": "sistrix crawler", + "addition_date": "2011/08/02", + "instances": [] + } + , + { + "pattern": "Ahrefs(Bot|SiteAudit)", + "addition_date": "2011/08/28", + "instances": [ + "Mozilla/5.0 (compatible; AhrefsBot/5.2; News; +http://ahrefs.com/robot/)", + "Mozilla/5.0 (compatible; AhrefsBot/5.2; +http://ahrefs.com/robot/)", + "Mozilla/5.0 (compatible; AhrefsSiteAudit/5.2; +http://ahrefs.com/robot/)" + ] + } + , + { + "pattern": "fuelbot", + "addition_date": "2018/06/28", + "instances": [ + "fuelbot" + ] + } + , + { + "pattern": "CrunchBot", + "addition_date": "2018/06/28", + "instances": [ + "CrunchBot/1.0 (+http://www.leadcrunch.com/crunchbot)" + ] + } + , + { + "pattern": "centurybot9", + "addition_date": "2018/06/28", + "instances": [ + "Mozilla/5.0 (compatible; Go-http-client/1.1; +centurybot9@gmail.com)" + ] + } + , + { + "pattern": "IndeedBot", + "addition_date": "2018/06/28", + "instances": [ + "Mozilla/5.0 (Windows NT 6.1; rv:38.0) Gecko/20100101 Firefox/38.0 (IndeedBot 1.1)" + ] + } + , + { + "pattern": "mappydata", + "addition_date": "2018/06/28", + "instances": [ + "Mozilla/5.0 (compatible; Mappy/1.0; +http://mappydata.net/bot/)" + ] + } + , + { + "pattern": "woobot", + "addition_date": "2018/06/28", + "instances": [ + "woobot" + ] + } + , + { + "pattern": "ZoominfoBot", + "addition_date": "2018/06/28", + "instances": [ + "ZoominfoBot (zoominfobot at zoominfo dot com)" + ] + } + , + { + "pattern": "PrivacyAwareBot", + "addition_date": "2018/06/28", + "instances": [ + "Mozilla/5.0 (compatible; PrivacyAwareBot/1.1; +http://www.privacyaware.org)" + ] + } + , + { + "pattern": "Multiviewbot", + "addition_date": "2018/06/28", + "instances": [ + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Multiviewbot" + ] + } + , + { + "pattern": "SWIMGBot", + "addition_date": "2018/06/28", + "instances": [ + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36 SWIMGBot" + ] + } + , + { + "pattern": "Grobbot", + "addition_date": "2018/06/28", + "instances": [ + "Mozilla/5.0 (compatible; Grobbot/2.2; +https://grob.it)" + ] + } + , + { + "pattern": "eright", + "addition_date": "2018/06/28", + "instances": [ + "Mozilla/5.0 (compatible; eright/1.0; +bot@eright.com)" + ] + } + , + { + "pattern": "Apercite", + "addition_date": "2018/06/28", + "instances": [ + "Mozilla/5.0 (compatible; Apercite; +http://www.apercite.fr/robot/index.html)" + ] + } + , + { + "pattern": "semanticbot", + "addition_date": "2018/06/28", + "instances": [ + "semanticbot", + "semanticbot (info@semanticaudience.com)" + ] + } + , + { + "pattern": "Aboundex", + "addition_date": "2011/09/28", + "url": "http://www.aboundex.com/crawler/", + "instances": [ + "Aboundex/0.2 (http://www.aboundex.com/crawler/)", + "Aboundex/0.3 (http://www.aboundex.com/crawler/)" + ] + } + , + { + "pattern": "domaincrawler", + "addition_date": "2011/10/21", + "instances": [ + "CipaCrawler/3.0 (info@domaincrawler.com; http://www.domaincrawler.com/www.example.com)" + ] + } + , + { + "pattern": "wbsearchbot", + "addition_date": "2011/12/21", + "url": "http://www.warebay.com/bot.html", + "instances": [] + } + , + { + "pattern": "summify", + "addition_date": "2012/01/04", + "url": "http://summify.com", + "instances": [ + "Summify (Summify/1.0.1; +http://summify.com)" + ] + } + , + { + "pattern": "CCBot", + "addition_date": "2012/02/05", + "url": "http://www.commoncrawl.org/bot.html", + "instances": [ + "CCBot/2.0 (http://commoncrawl.org/faq/)", + "CCBot/2.0 (https://commoncrawl.org/faq/)" + ] + } + , + { + "pattern": "edisterbot", + "addition_date": "2012/02/25", + "instances": [] + } + , + { + "pattern": "seznambot", + "addition_date": "2012/03/14", + "instances": [ + "Mozilla/5.0 (compatible; SeznamBot/3.2-test1-1; +http://napoveda.seznam.cz/en/seznambot-intro/)", + "Mozilla/5.0 (compatible; SeznamBot/3.2-test1; +http://napoveda.seznam.cz/en/seznambot-intro/)", + "Mozilla/5.0 (compatible; SeznamBot/3.2-test2; +http://napoveda.seznam.cz/en/seznambot-intro/)", + "Mozilla/5.0 (compatible; SeznamBot/3.2-test4; +http://napoveda.seznam.cz/en/seznambot-intro/)", + "Mozilla/5.0 (compatible; SeznamBot/3.2; +http://napoveda.seznam.cz/en/seznambot-intro/)" + ] + } + , + { + "pattern": "ec2linkfinder", + "addition_date": "2012/03/22", + "instances": [ + "ec2linkfinder" + ] + } + , + { + "pattern": "gslfbot", + "addition_date": "2012/04/03", + "instances": [] + } + , + { + "pattern": "aiHitBot", + "addition_date": "2012/04/16", + "instances": [ + "Mozilla/5.0 (compatible; aiHitBot/2.9; +https://www.aihitdata.com/about)" + ] + } + , + { + "pattern": "intelium_bot", + "addition_date": "2012/05/07", + "instances": [] + } + , + { + "pattern": "facebookexternalhit", + "addition_date": "2012/05/07", + "instances": [ + "facebookexternalhit/1.0 (+http://www.facebook.com/externalhit_uatext.php)", + "facebookexternalhit/1.1", + "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)" + ] + } + , + { + "pattern": "Yeti", + "addition_date": "2012/05/07", + "url": "http://naver.me/bot", + "instances": [ + "Mozilla/5.0 (compatible; Yeti/1.1; +http://naver.me/bot)" + ] + } + , + { + "pattern": "RetrevoPageAnalyzer", + "addition_date": "2012/05/07", + "instances": [ + "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; RetrevoPageAnalyzer; +http://www.retrevo.com/content/about-us)" + ] + } + , + { + "pattern": "lb-spider", + "addition_date": "2012/05/07", + "instances": [] + } + , + { + "pattern": "Sogou", + "addition_date": "2012/05/13", + "url": "http://www.sogou.com/docs/help/webmasters.htm#07", + "instances": [ + "Sogou News Spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)", + "Sogou Pic Spider/3.0(+http://www.sogou.com/docs/help/webmasters.htm#07)", + "Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)" + ] + } + , + { + "pattern": "lssbot", + "addition_date": "2012/05/15", + "instances": [] + } + , + { + "pattern": "careerbot", + "addition_date": "2012/05/23", + "url": "http://www.career-x.de/bot.html", + "instances": [] + } + , + { + "pattern": "wotbox", + "addition_date": "2012/06/12", + "url": "http://www.wotbox.com", + "instances": [ + "Wotbox/2.0 (bot@wotbox.com; http://www.wotbox.com)", + "Wotbox/2.01 (+http://www.wotbox.com/bot/)" + ] + } + , + { + "pattern": "wocbot", + "addition_date": "2012/07/25", + "url": "http://www.wocodi.com/crawler", + "instances": [] + } + , + { + "pattern": "ichiro", + "addition_date": "2012/08/28", + "url": "http://help.goo.ne.jp/help/article/1142", + "instances": [ + "DoCoMo/2.0 P900i(c100;TB;W24H11) (compatible; ichiro/mobile goo; +http://help.goo.ne.jp/help/article/1142/)", + "DoCoMo/2.0 P900i(c100;TB;W24H11) (compatible; ichiro/mobile goo; +http://search.goo.ne.jp/option/use/sub4/sub4-1/)", + "DoCoMo/2.0 P900i(c100;TB;W24H11) (compatible; ichiro/mobile goo;+http://search.goo.ne.jp/option/use/sub4/sub4-1/)", + "DoCoMo/2.0 P900i(c100;TB;W24H11)(compatible; ichiro/mobile goo;+http://help.goo.ne.jp/door/crawler.html)", + "DoCoMo/2.0 P901i(c100;TB;W24H11) (compatible; ichiro/mobile goo; +http://help.goo.ne.jp/door/crawler.html)", + "KDDI-CA31 UP.Browser/6.2.0.7.3.129 (GUI) MMP/2.0 (compatible; ichiro/mobile goo; +http://help.goo.ne.jp/help/article/1142/)", + "KDDI-CA31 UP.Browser/6.2.0.7.3.129 (GUI) MMP/2.0 (compatible; ichiro/mobile goo; +http://search.goo.ne.jp/option/use/sub4/sub4-1/)", + "KDDI-CA31 UP.Browser/6.2.0.7.3.129 (GUI) MMP/2.0 (compatible; ichiro/mobile goo;+http://search.goo.ne.jp/option/use/sub4/sub4-1/)", + "ichiro/2.0 (http://help.goo.ne.jp/door/crawler.html)", + "ichiro/2.0 (ichiro@nttr.co.jp)", + "ichiro/3.0 (http://help.goo.ne.jp/door/crawler.html)", + "ichiro/3.0 (http://help.goo.ne.jp/help/article/1142)", + "ichiro/3.0 (http://search.goo.ne.jp/option/use/sub4/sub4-1/)", + "ichiro/4.0 (http://help.goo.ne.jp/door/crawler.html)", + "ichiro/5.0 (http://help.goo.ne.jp/door/crawler.html)" + ] + } + , + { + "pattern": "DuckDuckBot", + "addition_date": "2012/09/19", + "url": "http://duckduckgo.com/duckduckbot.html", + "instances": [ + "DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)", + "DuckDuckBot/1.1; (+http://duckduckgo.com/duckduckbot.html)" + ] + } + , + { + "pattern": "lssrocketcrawler", + "addition_date": "2012/09/24", + "instances": [] + } + , + { + "pattern": "drupact", + "addition_date": "2012/09/27", + "url": "http://www.arocom.de/drupact", + "instances": [ + "drupact/0.7; http://www.arocom.de/drupact" + ] + } + , + { + "pattern": "webcompanycrawler", + "addition_date": "2012/10/03", + "instances": [] + } + , + { + "pattern": "acoonbot", + "addition_date": "2012/10/07", + "url": "http://www.acoon.de/robot.asp", + "instances": [] + } + , + { + "pattern": "openindexspider", + "addition_date": "2012/10/26", + "url": "http://www.openindex.io/en/webmasters/spider.html", + "instances": [] + } + , + { + "pattern": "gnam gnam spider", + "addition_date": "2012/10/31", + "instances": [] + } + , + { + "pattern": "web-archive-net.com.bot", + "instances": [] + } + , + { + "pattern": "backlinkcrawler", + "addition_date": "2013/01/04", + "instances": [] + } + , + { + "pattern": "coccoc", + "addition_date": "2013/01/04", + "url": "http://help.coccoc.vn/", + "instances": [ + "Mozilla/5.0 (compatible; coccoc/1.0; +http://help.coccoc.com/)", + "Mozilla/5.0 (compatible; coccoc/1.0; +http://help.coccoc.com/searchengine)", + "Mozilla/5.0 (compatible; coccocbot-image/1.0; +http://help.coccoc.com/searchengine)", + "Mozilla/5.0 (compatible; coccocbot-web/1.0; +http://help.coccoc.com/searchengine)", + "Mozilla/5.0 (compatible; image.coccoc/1.0; +http://help.coccoc.com/)", + "Mozilla/5.0 (compatible; imagecoccoc/1.0; +http://help.coccoc.com/)", + "Mozilla/5.0 (compatible; imagecoccoc/1.0; +http://help.coccoc.com/searchengine)", + "coccoc", + "coccoc/1.0 ()", + "coccoc/1.0 (http://help.coccoc.com/)", + "coccoc/1.0 (http://help.coccoc.vn/)" + ] + } + , + { + "pattern": "integromedb", + "addition_date": "2013/01/10", + "url": "http://www.integromedb.org/Crawler", + "instances": [ + "www.integromedb.org/Crawler" + ] + } + , + { + "pattern": "content crawler spider", + "addition_date": "2013/01/11", + "instances": [] + } + , + { + "pattern": "toplistbot", + "addition_date": "2013/02/05", + "instances": [] + } + , + { + "pattern": "it2media-domain-crawler", + "addition_date": "2013/03/12", + "instances": [ + "it2media-domain-crawler/1.0 on crawler-prod.it2media.de", + "it2media-domain-crawler/2.0" + ] + } + , + { + "pattern": "ip-web-crawler.com", + "addition_date": "2013/03/22", + "instances": [] + } + , + { + "pattern": "siteexplorer.info", + "addition_date": "2013/05/01", + "instances": [ + "Mozilla/5.0 (compatible; SiteExplorer/1.0b; +http://siteexplorer.info/)", + "Mozilla/5.0 (compatible; SiteExplorer/1.1b; +http://siteexplorer.info/Backlink-Checker-Spider/)" + ] + } + , + { + "pattern": "elisabot", + "addition_date": "2013/06/27", + "instances": [] + } + , + { + "pattern": "proximic", + "addition_date": "2013/09/12", + "url": "http://www.proximic.com/info/spider.php", + "instances": [ + "Mozilla/5.0 (compatible; proximic; +http://www.proximic.com)", + "Mozilla/5.0 (compatible; proximic; +http://www.proximic.com/info/spider.php)" + ] + } + , + { + "pattern": "changedetection", + "addition_date": "2013/09/13", + "url": "http://www.changedetection.com/bot.html", + "instances": [ + "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; http://www.changedetection.com/bot.html )" + ] + } + , + { + "pattern": "arabot", + "addition_date": "2013/10/09", + "instances": [] + } + , + { + "pattern": "WeSEE:Search", + "addition_date": "2013/11/18", + "instances": [ + "WeSEE:Search", + "WeSEE:Search/0.1 (Alpha, http://www.wesee.com/en/support/bot/)" + ] + } + , + { + "pattern": "niki-bot", + "addition_date": "2014/01/01", + "instances": [] + } + , + { + "pattern": "CrystalSemanticsBot", + "addition_date": "2014/02/17", + "url": "http://www.crystalsemantics.com/user-agent/", + "instances": [] + } + , + { + "pattern": "rogerbot", + "addition_date": "2014/02/28", + "url": "http://moz.com/help/pro/what-is-rogerbot-", + "instances": [ + "Mozilla/5.0 (compatible; rogerBot/1.0; UrlCrawler; http://www.seomoz.org/dp/rogerbot)", + "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+partager@moz.com)", + "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+shiny@moz.com)", + "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-wherecat@moz.com", + "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-wherecat@moz.com)", + "rogerbot/1.0 (http://www.moz.com/dp/rogerbot, rogerbot-crawler@moz.com)", + "rogerbot/1.0 (http://www.seomoz.org/dp/rogerbot, rogerbot-crawler+shiny@seomoz.org)", + "rogerbot/1.0 (http://www.seomoz.org/dp/rogerbot, rogerbot-crawler@seomoz.org)", + "rogerbot/1.0 (http://www.seomoz.org/dp/rogerbot, rogerbot-wherecat@moz.com)", + "rogerbot/1.1 (http://moz.com/help/guides/search-overview/crawl-diagnostics#more-help, rogerbot-crawler+pr2-crawler-05@moz.com)", + "rogerbot/1.1 (http://moz.com/help/guides/search-overview/crawl-diagnostics#more-help, rogerbot-crawler+pr4-crawler-11@moz.com)", + "rogerbot/1.1 (http://moz.com/help/guides/search-overview/crawl-diagnostics#more-help, rogerbot-crawler+pr4-crawler-15@moz.com)", + "rogerbot/1.2 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+phaser-testing-crawler-01@moz.com)" + ] + } + , + { + "pattern": "360Spider", + "addition_date": "2014/03/14", + "url": "http://needs-be.blogspot.co.uk/2013/02/how-to-block-spider360.html", + "instances": [ + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1; 360Spider", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1; 360Spider(compatible; HaosouSpider; http://www.haosou.com/help/help_3_2.html)", + "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36 QIHU 360SE; 360Spider", + "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; ) Firefox/1.5.0.11; 360Spider", + "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.8.0.11) Firefox/1.5.0.11; 360Spider", + "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.8.0.11) Firefox/1.5.0.11 360Spider;", + "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.8.0.11) Gecko/20070312 Firefox/1.5.0.11; 360Spider", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider(compatible; HaosouSpider; http://www.haosou.com/help/help_3_2.html)" + ] + } + , + { + "pattern": "psbot", + "addition_date": "2014/03/31", + "url": "http://www.picsearch.com/bot.html", + "instances": [ + "psbot-image (+http://www.picsearch.com/bot.html)", + "psbot-page (+http://www.picsearch.com/bot.html)", + "psbot/0.1 (+http://www.picsearch.com/bot.html)" + ] + } + , + { + "pattern": "InterfaxScanBot", + "addition_date": "2014/03/31", + "url": "http://scan-interfax.ru", + "instances": [] + } + , + { + "pattern": "CC Metadata Scaper", + "addition_date": "2014/04/01", + "url": "http://wiki.creativecommons.org/Metadata_Scraper", + "instances": [ + "CC Metadata Scaper http://wiki.creativecommons.org/Metadata_Scraper" + ] + } + , + { + "pattern": "g00g1e.net", + "addition_date": "2014/04/01", + "url": "http://www.g00g1e.net/", + "instances": [] + } + , + { + "pattern": "GrapeshotCrawler", + "addition_date": "2014/04/01", + "url": "http://www.grapeshot.co.uk/crawler.php", + "instances": [ + "Mozilla/5.0 (compatible; GrapeshotCrawler/2.0; +http://www.grapeshot.co.uk/crawler.php)" + ] + } + , + { + "pattern": "urlappendbot", + "addition_date": "2014/05/10", + "url": "http://www.profound.net/urlappendbot.html", + "instances": [ + "Mozilla/5.0 (compatible; URLAppendBot/1.0; +http://www.profound.net/urlappendbot.html)" + ] + } + , + { + "pattern": "brainobot", + "addition_date": "2014/06/24", + "instances": [] + } + , + { + "pattern": "fr-crawler", + "addition_date": "2014/07/31", + "instances": [ + "Mozilla/5.0 (compatible; fr-crawler/1.1)" + ] + } + , + { + "pattern": "binlar", + "addition_date": "2014/09/12", + "instances": [ + "binlar_2.6.3 binlar2.6.3@unspecified.mail", + "binlar_2.6.3 binlar_2.6.3@unspecified.mail", + "binlar_2.6.3 larbin2.6.3@unspecified.mail", + "binlar_2.6.3 phanendra_kalapala@McAfee.com", + "binlar_2.6.3 test@mgmt.mic" + ] + } + , + { + "pattern": "SimpleCrawler", + "addition_date": "2014/09/12", + "instances": [ + "SimpleCrawler/0.1" + ] + } + , + { + "pattern": "Twitterbot", + "addition_date": "2014/09/12", + "url": "https://dev.twitter.com/cards/getting-started", + "instances": [ + "Twitterbot/0.1", + "Twitterbot/1.0" + ] + } + , + { + "pattern": "cXensebot", + "addition_date": "2014/10/05", + "instances": [ + "cXensebot/1.1a" + ], + "url": "http://www.cxense.com/bot.html" + } + , + { + "pattern": "smtbot", + "addition_date": "2014/10/04", + "instances": [ + "Mozilla/5.0 (compatible; SMTBot/1.0; +http://www.similartech.com/smtbot)", + "SMTBot (similartech.com/smtbot)" + ], + "url": "http://www.similartech.com/smtbot" + } + , + { + "pattern": "bnf.fr_bot", + "addition_date": "2014/11/18", + "url": "http://www.bnf.fr/fr/outils/a.dl_web_capture_robot.html", + "instances": [ + "Mozilla/5.0 (compatible; bnf.fr_bot; +http://www.bnf.fr/fr/outils/a.dl_web_capture_robot.html)" + ] + } + , + { + "pattern": "A6-Indexer", + "addition_date": "2014/12/05", + "url": "http://www.a6corp.com/a6-web-scraping-policy/", + "instances": [ + "A6-Indexer" + ] + } + , + { + "pattern": "ADmantX", + "addition_date": "2014/12/05", + "url": "http://www.admantx.com", + "instances": [ + "ADmantX Platform Semantic Analyzer - ADmantX Inc. - www.admantx.com - support@admantx.com" + ] + } + , + { + "pattern": "Facebot", + "url": "https://developers.facebook.com/docs/sharing/best-practices#crawl", + "addition_date": "2014/12/30", + "instances": [ + "Facebot/1.0" + ] + } + , + { + "pattern": "OrangeBot\\/", + "instances": [ + "Mozilla/5.0 (compatible; OrangeBot/2.0; support.orangebot@orange.com" + ], + "addition_date": "2015/01/12" + } + , + { + "pattern": "memorybot", + "url": "http://mignify.com/bot.htm", + "instances": [ + "Mozilla/5.0 (compatible; memorybot/1.21.14 +http://mignify.com/bot.html)" + ], + "addition_date": "2015/02/01" + } + , + { + "pattern": "AdvBot", + "url": "http://advbot.net/bot.html", + "instances": [ + "Mozilla/5.0 (compatible; AdvBot/2.0; +http://advbot.net/bot.html)" + ], + "addition_date": "2015/02/01" + } + , + { + "pattern": "MegaIndex", + "url": "https://www.megaindex.ru/?tab=linkAnalyze", + "instances": [ + "Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +https://www.megaindex.ru/?tab=linkAnalyze)" + ], + "addition_date": "2015/03/28" + } + , + { + "pattern": "SemanticScholarBot", + "url": "http://s2.allenai.org/bot.html", + "instances": [ + "SemanticScholarBot/1.0 (+http://s2.allenai.org/bot.html)" + ], + "addition_date": "2015/03/28" + } + , + { + "pattern": "ltx71", + "url": "http://ltx71.com/", + "instances": [ + "ltx71 - (http://ltx71.com/)" + ], + "addition_date": "2015/04/04" + } + , + { + "pattern": "nerdybot", + "url": "http://nerdybot.com/", + "instances": [ + "nerdybot" + ], + "addition_date": "2015/04/05" + } + , + { + "pattern": "xovibot", + "url": "http://www.xovibot.net/", + "instances": [ + "Mozilla/5.0 (compatible; XoviBot/2.0; +http://www.xovibot.net/)" + ], + "addition_date": "2015/04/05" + } + , + { + "pattern": "BUbiNG", + "url": "http://law.di.unimi.it/BUbiNG.html", + "instances": [ + "BUbiNG (+http://law.di.unimi.it/BUbiNG.html)" + ], + "addition_date": "2015/04/06" + } + , + { + "pattern": "Qwantify", + "url": "https://www.qwant.com/", + "instances": [ + "Mozilla/5.0 (compatible; Qwantify/2.0n; +https://www.qwant.com/)/*", + "Mozilla/5.0 (compatible; Qwantify/2.4w; +https://www.qwant.com/)/2.4w" + ], + "addition_date": "2015/04/06" + } + , + { + "pattern": "archive.org_bot", + "url": "http://www.archive.org/details/archive.org_bot", + "instances": [ + "Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot)" + ], + "addition_date": "2015/04/14" + } + , + { + "pattern": "Applebot", + "url": "http://www.apple.com/go/applebot", + "addition_date": "2015/04/15", + "instances": [ + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1)", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1; +http://www.apple.com/go/applebot)", + "Mozilla/5.0 (compatible; Applebot/0.3; +http://www.apple.com/go/applebot)", + "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Applebot/0.3; +http://www.apple.com/go/applebot)", + "Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B410 Safari/600.1.4 (Applebot/0.1; +http://www.apple.com/go/applebot)" + ] + } + , + { + "pattern": "TweetmemeBot", + "url": "http://datasift.com/bot.html", + "instances": [ + "Mozilla/5.0 (TweetmemeBot/4.0; +http://datasift.com/bot.html) Gecko/20100101 Firefox/31.0" + ], + "addition_date": "2015/04/15" + } + , + { + "pattern": "crawler4j", + "url": "https://github.com/yasserg/crawler4j", + "instances": [ + "crawler4j (http://code.google.com/p/crawler4j/)" + ], + "addition_date": "2015/05/07" + } + , + { + "pattern": "findxbot", + "url": "http://www.findxbot.com", + "instances": [ + "Mozilla/5.0 (compatible; Findxbot/1.0; +http://www.findxbot.com)" + ], + "addition_date": "2015/05/07" + } + , + { + "pattern": "S[eE][mM]rushBot", + "url": "http://www.semrush.com/bot.html", + "instances": [ + "Mozilla/5.0 (compatible; SemrushBot/0.98~bl; +http://www.semrush.com/bot.html)", + "SEMrushBot" + ], + "addition_date": "2015/05/26" + } + , + { + "pattern": "yoozBot", + "url": "http://yooz.ir", + "instances": [ + "Mozilla/5.0 (compatible; yoozBot-2.2; http://yooz.ir; info@yooz.ir)" + ], + "addition_date": "2015/05/26" + } + , + { + "pattern": "lipperhey", + "url": "http://www.lipperhey.com/", + "instances": [ + "Mozilla/5.0 (compatible; Lipperhey Link Explorer; http://www.lipperhey.com/)", + "Mozilla/5.0 (compatible; Lipperhey SEO Service; http://www.lipperhey.com/)", + "Mozilla/5.0 (compatible; Lipperhey Site Explorer; http://www.lipperhey.com/)", + "Mozilla/5.0 (compatible; Lipperhey-Kaus-Australis/5.0; +https://www.lipperhey.com/en/about/)" + ], + "addition_date": "2015/08/26" + } + , + { + "pattern": "Y!J", + "url": "https://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/~/%E3%82%A6%E3%82%A7%E3%83%96%E3%83%9A%E3%83%BC%E3%82%B8%E3%81%AB%E3%82%A2%E3%82%AF%E3%82%BB%E3%82%B9%E3%81%99%E3%82%8B%E3%82%B7%E3%82%B9%E3%83%86%E3%83%A0%E3%81%AE%E3%83%A6%E3%83%BC%E3%82%B6%E3%83%BC%E3%82%A8%E3%83%BC%E3%82%B8%E3%82%A7%E3%83%B3%E3%83%88%E3%81%AB%E3%81%A4%E3%81%84%E3%81%A6", + "instances": [ + "Y!J-ASR/0.1 crawler (http://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/)", + "Y!J-BRJ/YATS crawler (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)", + "Y!J-PSC/1.0 crawler (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)", + "Y!J-BRW/1.0 crawler (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)", + "Mozilla/5.0 (iPhone; Y!J-BRY/YATSH crawler; http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)", + "Mozilla/5.0 (compatible; Y!J SearchMonkey/1.0 (Y!J-AGENT; http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html))" + ], + "addition_date": "2015/05/26" + } + , + { + "pattern": "Domain Re-Animator Bot", + "url": "http://domainreanimator.com", + "instances": [ + "Domain Re-Animator Bot (http://domainreanimator.com) - support@domainreanimator.com" + ], + "addition_date": "2015/04/14" + } + , + { + "pattern": "AddThis", + "url": "https://www.addthis.com", + "instances": [ + "AddThis.com robot tech.support@clearspring.com" + ], + "addition_date": "2015/06/02" + } + , + { + "pattern": "Screaming Frog SEO Spider", + "url": "http://www.screamingfrog.co.uk/seo-spider", + "instances": [ + "Screaming Frog SEO Spider/5.1" + ], + "addition_date": "2016/01/08" + } + , + { + "pattern": "MetaURI", + "url": "http://www.useragentstring.com/MetaURI_id_17683.php", + "instances": [ + "MetaURI API/2.0 +metauri.com" + ], + "addition_date": "2016/01/02" + } + , + { + "pattern": "Scrapy", + "url": "http://scrapy.org/", + "instances": [ + "Scrapy/1.0.3 (+http://scrapy.org)" + ], + "addition_date": "2016/01/02" + } + , + { + "pattern": "Livelap[bB]ot", + "url": "http://site.livelap.com/crawler", + "instances": [ + "LivelapBot/0.2 (http://site.livelap.com/crawler)", + "Livelapbot/0.1" + ], + "addition_date": "2016/01/02" + } + , + { + "pattern": "OpenHoseBot", + "url": "http://www.openhose.org/bot.html", + "instances": [ + "Mozilla/5.0 (compatible; OpenHoseBot/2.1; +http://www.openhose.org/bot.html)" + ], + "addition_date": "2016/01/02" + } + , + { + "pattern": "CapsuleChecker", + "url": "http://www.capsulink.com/about", + "instances": [ + "CapsuleChecker (http://www.capsulink.com/)" + ], + "addition_date": "2016/01/02" + } + , + { + "pattern": "collection@infegy.com", + "url": "http://infegy.com/", + "instances": [ + "Mozilla/5.0 (compatible) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.73 Safari/537.36 collection@infegy.com" + ], + "addition_date": "2016/01/03" + } + , + { + "pattern": "IstellaBot", + "url": "http://www.tiscali.it/", + "instances": [ + "Mozilla/5.0 (compatible; IstellaBot/1.23.15 +http://www.tiscali.it/)" + ], + "addition_date": "2016/01/09" + } + , + { + "pattern": "DeuSu\\/", + "addition_date": "2016/01/23", + "url": "https://deusu.de/robot.html", + "instances": [ + "Mozilla/5.0 (compatible; DeuSu/0.1.0; +https://deusu.org)", + "Mozilla/5.0 (compatible; DeuSu/5.0.2; +https://deusu.de/robot.html)" + ] + } + , + { + "pattern": "betaBot", + "addition_date": "2016/01/23", + "instances": [] + } + , + { + "pattern": "Cliqzbot\\/", + "addition_date": "2016/01/23", + "url": "http://cliqz.com/company/cliqzbot", + "instances": [ + "Cliqzbot/0.1 (+http://cliqz.com +cliqzbot@cliqz.com)", + "Cliqzbot/0.1 (+http://cliqz.com/company/cliqzbot)", + "Mozilla/5.0 (compatible; Cliqzbot/0.1 +http://cliqz.com/company/cliqzbot)", + "Mozilla/5.0 (compatible; Cliqzbot/1.0 +http://cliqz.com/company/cliqzbot)" + ] + } + , + { + "pattern": "MojeekBot\\/", + "addition_date": "2016/01/23", + "url": "https://www.mojeek.com/bot.html", + "instances": [ + "MojeekBot/0.2 (archi; http://www.mojeek.com/bot.html)", + "Mozilla/5.0 (compatible; MojeekBot/0.2; http://www.mojeek.com/bot.html#relaunch)", + "Mozilla/5.0 (compatible; MojeekBot/0.2; http://www.mojeek.com/bot.html)", + "Mozilla/5.0 (compatible; MojeekBot/0.5; http://www.mojeek.com/bot.html)", + "Mozilla/5.0 (compatible; MojeekBot/0.6; +https://www.mojeek.com/bot.html)", + "Mozilla/5.0 (compatible; MojeekBot/0.6; http://www.mojeek.com/bot.html)" + ] + } + , + { + "pattern": "netEstate NE Crawler", + "addition_date": "2016/01/23", + "url": "+http://www.website-datenbank.de/", + "instances": [ + "netEstate NE Crawler (+http://www.sengine.info/)", + "netEstate NE Crawler (+http://www.website-datenbank.de/)" + ] + } + , + { + "pattern": "SafeSearch microdata crawler", + "addition_date": "2016/01/23", + "url": "https://safesearch.avira.com", + "instances": [ + "SafeSearch microdata crawler (https://safesearch.avira.com, safesearch-abuse@avira.com)" + ] + } + , + { + "pattern": "Gluten Free Crawler\\/", + "addition_date": "2016/01/23", + "url": "http://glutenfreepleasure.com/", + "instances": [ + "Mozilla/5.0 (compatible; Gluten Free Crawler/1.0; +http://glutenfreepleasure.com/)" + ] + } + , + { + "pattern": "Sonic", + "addition_date": "2016/02/08", + "url": "http://www.yama.info.waseda.ac.jp/~crawler/info.html", + "instances": [ + "Mozilla/5.0 (compatible; RankSonicSiteAuditor/1.0; +https://ranksonic.com/ranksonic_sab.html)", + "Mozilla/5.0 (compatible; Sonic/1.0; http://www.yama.info.waseda.ac.jp/~crawler/info.html)", + "Mozzila/5.0 (compatible; Sonic/1.0; http://www.yama.info.waseda.ac.jp/~crawler/info.html)" + ] + } + , + { + "pattern": "Sysomos", + "addition_date": "2016/02/08", + "url": "http://www.sysomos.com", + "instances": [ + "Mozilla/5.0 (compatible; Sysomos/1.0; +http://www.sysomos.com/; Sysomos)" + ] + } + , + { + "pattern": "Trove", + "addition_date": "2016/02/08", + "url": "http://www.trove.com", + "instances": [] + } + , + { + "pattern": "deadlinkchecker", + "addition_date": "2016/02/08", + "url": "http://www.deadlinkchecker.com", + "instances": [ + "www.deadlinkchecker.com Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36", + "www.deadlinkchecker.com XMLHTTP/1.0", + "www.deadlinkchecker.com XMLHTTP/1.0 Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36" + ] + } + , + { + "pattern": "Slack-ImgProxy", + "addition_date": "2016/04/25", + "url": "https://api.slack.com/robots", + "instances": [ + "Slack-ImgProxy (+https://api.slack.com/robots)", + "Slack-ImgProxy 0.59 (+https://api.slack.com/robots)", + "Slack-ImgProxy 0.66 (+https://api.slack.com/robots)", + "Slack-ImgProxy 1.106 (+https://api.slack.com/robots)", + "Slack-ImgProxy 1.138 (+https://api.slack.com/robots)", + "Slack-ImgProxy 149 (+https://api.slack.com/robots)" + ] + } + , + { + "pattern": "Embedly", + "addition_date": "2016/04/25", + "url": "http://support.embed.ly", + "instances": [ + "Embedly +support@embed.ly", + "Mozilla/5.0 (compatible; Embedly/0.2; +http://support.embed.ly/)", + "Mozilla/5.0 (compatible; Embedly/0.2; snap; +http://support.embed.ly/)" + ] + } + , + { + "pattern": "RankActiveLinkBot", + "addition_date": "2016/06/20", + "url": "https://rankactive.com/resources/rankactive-linkbot", + "instances": [ + "Mozilla/5.0 (compatible; RankActiveLinkBot; +https://rankactive.com/resources/rankactive-linkbot)" + ] + } + , + { + "pattern": "iskanie", + "addition_date": "2016/09/02", + "url": "http://www.iskanie.com", + "instances": [ + "iskanie (+http://www.iskanie.com)" + ] + } + , + { + "pattern": "SafeDNSBot", + "addition_date": "2016/09/10", + "url": "https://www.safedns.com/searchbot", + "instances": [ + "SafeDNSBot (https://www.safedns.com/searchbot)" + ] + } + , + { + "pattern": "SkypeUriPreview", + "addition_date": "2016/10/10", + "instances": [ + "Mozilla/5.0 (Windows NT 6.1; WOW64) SkypeUriPreview Preview/0.5" + ] + } + , + { + "pattern": "Veoozbot", + "addition_date": "2016/11/03", + "url": "http://www.veooz.com/veoozbot.html", + "instances": [ + "Mozilla/5.0 (compatible; Veoozbot/1.0; +http://www.veooz.com/veoozbot.html)" + ] + } + , + { + "pattern": "Slackbot", + "addition_date": "2016/11/03", + "url": "https://api.slack.com/robots", + "instances": [ + "Slackbot-LinkExpanding (+https://api.slack.com/robots)", + "Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)" + ] + } + , + { + "pattern": "redditbot", + "addition_date": "2016/11/03", + "url": "http://www.reddit.com/feedback", + "instances": [ + "Mozilla/5.0 (compatible; redditbot/1.0; +http://www.reddit.com/feedback)" + ] + } + , + { + "pattern": "datagnionbot", + "addition_date": "2016/11/03", + "url": "http://www.datagnion.com/bot.html", + "instances": [ + "datagnionbot (+http://www.datagnion.com/bot.html)" + ] + } + , + { + "pattern": "Google-Adwords-Instant", + "addition_date": "2016/11/03", + "url": "http://www.google.com/adsbot.html", + "instances": [ + "Google-Adwords-Instant (+http://www.google.com/adsbot.html)" + ] + } + , + { + "pattern": "adbeat_bot", + "addition_date": "2016/11/04", + "instances": [ + "Mozilla/5.0 (compatible; adbeat_bot; +support@adbeat.com; support@adbeat.com)", + "adbeat_bot" + ] + } + , + { + "pattern": "WhatsApp", + "addition_date": "2016/11/15", + "url": "https://www.whatsapp.com/", + "instances": [ + "WhatsApp", + "WhatsApp/2.12.15/i", + "WhatsApp/2.12.16/i", + "WhatsApp/2.12.17/i", + "WhatsApp/2.12.449 A", + "WhatsApp/2.12.453 A", + "WhatsApp/2.12.510 A", + "WhatsApp/2.12.540 A", + "WhatsApp/2.12.548 A", + "WhatsApp/2.12.555 A", + "WhatsApp/2.12.556 A", + "WhatsApp/2.16.1/i", + "WhatsApp/2.16.13 A", + "WhatsApp/2.16.2/i", + "WhatsApp/2.16.42 A", + "WhatsApp/2.16.57 A" + ] + } + , + { + "pattern": "contxbot", + "addition_date": "2017/02/25", + "instances": [ + "Mozilla/5.0 (compatible;contxbot/1.0)" + ] + } + , + { + "pattern": "pinterest.com.bot", + "addition_date": "2017/03/03", + "instances": [ + "Pinterest/0.2 (+http://www.pinterest.com/bot.html)" + ], + "url": "http://www.pinterest.com/bot.html" + } + , + { + "pattern": "electricmonk", + "addition_date": "2017/03/04", + "instances": [ + "Mozilla/5.0 (compatible; electricmonk/3.2.0 +https://www.duedil.com/our-crawler/)" + ], + "url": "https://www.duedil.com/our-crawler/" + } + , + { + "pattern": "GarlikCrawler", + "addition_date": "2017/03/18", + "instances": [ + "GarlikCrawler/1.2 (http://garlik.com/, crawler@garlik.com)" + ], + "url": "http://garlik.com/" + } + , + { + "pattern": "BingPreview\\/", + "addition_date": "2017/04/23", + "url": "https://www.bing.com/webmaster/help/which-crawlers-does-bing-use-8c184ec0", + "instances": [ + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534+ (KHTML, like Gecko) BingPreview/1.0b", + "Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0; BingPreview/1.0b) like Gecko", + "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0; WOW64; Trident/6.0; BingPreview/1.0b)", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; WOW64; Trident/5.0; BingPreview/1.0b)", + "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 BingPreview/1.0b" + ] + } + , + { + "pattern": "vebidoobot", + "addition_date": "2017/05/08", + "instances": [ + "Mozilla/5.0 (compatible; vebidoobot/1.0; +https://blog.vebidoo.de/vebidoobot/" + ], + "url": "https://blog.vebidoo.de/vebidoobot/" + } + , + { + "pattern": "FemtosearchBot", + "addition_date": "2017/05/16", + "instances": [ + "Mozilla/5.0 (compatible; FemtosearchBot/1.0; http://femtosearch.com)" + ], + "url": "http://femtosearch.com" + } + , + { + "pattern": "Yahoo Link Preview", + "addition_date": "2017/06/28", + "instances": [ + "Mozilla/5.0 (compatible; Yahoo Link Preview; https://help.yahoo.com/kb/mail/yahoo-link-preview-SLN23615.html)" + ], + "url": "https://help.yahoo.com/kb/mail/yahoo-link-preview-SLN23615.html" + } + , + { + "pattern": "MetaJobBot", + "addition_date": "2017/08/16", + "instances": [ + "Mozilla/5.0 (compatible; MetaJobBot; http://www.metajob.de/crawler)" + ], + "url": "http://www.metajob.de/the/crawler" + } + , + { + "pattern": "DomainStatsBot", + "addition_date": "2017/08/16", + "instances": [ + "DomainStatsBot/1.0 (http://domainstats.io/our-bot)" + ], + "url": "http://domainstats.io/our-bot" + } + , + { + "pattern": "mindUpBot", + "addition_date": "2017/08/16", + "instances": [ + "mindUpBot (datenbutler.de)" + ], + "url": "http://www.datenbutler.de/" + } + , + { + "pattern": "Daum\\/", + "addition_date": "2017/08/16", + "instances": [ + "Mozilla/5.0 (compatible; Daum/4.1; +http://cs.daum.net/faq/15/4118.html?faqId=28966)" + ], + "url": "http://cs.daum.net/faq/15/4118.html?faqId=28966" + } + , + { + "pattern": "Jugendschutzprogramm-Crawler", + "addition_date": "2017/08/16", + "instances": [ + "Jugendschutzprogramm-Crawler; Info: http://www.jugendschutzprogramm.de" + ], + "url": "http://www.jugendschutzprogramm.de" + } + , + { + "pattern": "Xenu Link Sleuth", + "addition_date": "2017/08/19", + "instances": [ + "Xenu Link Sleuth/1.3.8" + ], + "url": "http://home.snafu.de/tilman/xenulink.html" + } + , + { + "pattern": "Pcore-HTTP", + "addition_date": "2017/08/19", + "instances": [ + "Pcore-HTTP/v0.40.3" + ], + "url": "https://bitbucket.org/softvisio/pcore/overview" + } + , + { + "pattern": "moatbot", + "addition_date": "2017/09/16", + "instances": [ + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.111 Safari/537.36 moatbot", + "Mozilla/5.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/600.1.3 (KHTML, like Gecko) Version/8.0 Mobile/12A4345d Safari/600.1.4 moatbot" + ], + "url": "https://moat.com" + } + , + { + "pattern": "KosmioBot", + "addition_date": "2017/09/16", + "instances": [ + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.125 Safari/537.36 (compatible; KosmioBot/1.0; +http://kosm.io/bot.html)" + ], + "url": "http://kosm.io/bot.html" + } + , + { + "pattern": "pingdom", + "addition_date": "2017/09/16", + "instances": [ + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/59.0.3071.109 Chrome/59.0.3071.109 Safari/537.36 PingdomPageSpeed/1.0 (pingbot/2.0; +http://www.pingdom.com/)", + "Mozilla/5.0 (compatible; pingbot/2.0; +http://www.pingdom.com/)" + ], + "url": "http://www.pingdom.com" + } + , + { + "pattern": "PhantomJS", + "addition_date": "2017/09/18", + "instances": [ + "Mozilla/5.0 (Unknown; Linux x86_64) AppleWebKit/538.1 (KHTML, like Gecko) PhantomJS/2.1.1 Safari/538.1 bl.uk_lddc_renderbot/2.0.0 (+ http://www.bl.uk/aboutus/legaldeposit/websites/websites/faqswebmaster/index.html)" + ], + "url": "http://phantomjs.org/" + } + , + { + "pattern": "Gowikibot", + "addition_date": "2017/10/26", + "instances": [ + "Mozilla/5.0 (compatible; Gowikibot/1.0; +http://www.gowikibot.com)" + ], + "url": "http://www.gowikibot.com" + } + , + { + "pattern": "PiplBot", + "addition_date": "2017/10/30", + "instances": [ + "Mozilla/5.0+(compatible;+PiplBot;+http://www.pipl.com/bot/)" + ], + "url": "http://www.pipl.com/bot/" + } + , + { + "pattern": "Discordbot", + "addition_date": "2017/09/22", + "url": "https://discordapp.com", + "instances": [ + "Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)" + ] + } + , + { + "pattern": "TelegramBot", + "addition_date": "2017/10/01", + "instances": [ + "TelegramBot (like TwitterBot)" + ] + } + , + { + "pattern": "Jetslide", + "addition_date": "2017/09/27", + "url": "http://jetsli.de/crawler", + "instances": [ + "Mozilla/5.0 (compatible; Jetslide; +http://jetsli.de/crawler)" + ] + } + , + { + "pattern": "newsharecounts", + "addition_date": "2017/09/30", + "url": "http://newsharecounts.com/crawler", + "instances": [ + "Mozilla/5.0 (compatible; NewShareCounts.com/1.0; +http://newsharecounts.com/crawler)" + ] + } + , + { + "pattern": "James BOT", + "addition_date": "2017/10/12", + "url": "http://cognitiveseo.com/bot.html", + "instances": [ + "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6 - James BOT - WebCrawler http://cognitiveseo.com/bot.html" + ] + } + , + { + "pattern": "Barkrowler", + "addition_date": "2017/10/09", + "url": "http://www.exensa.com/crawl", + "instances": [ + "Barkrowler/0.5.1 (experimenting / debugging - sorry for your logs ) http://www.exensa.com/crawl - admin@exensa.com -- based on BuBiNG", + "Barkrowler/0.7 (+http://www.exensa.com/crawl)" + ] + } + , + { + "pattern": "TinEye", + "addition_date": "2017/10/14", + "url": "http://www.tineye.com/crawler.html", + "instances": [ + "Mozilla/5.0 (compatible; TinEye-bot/1.31; +http://www.tineye.com/crawler.html)", + "TinEye/1.1 (http://tineye.com/crawler.html)" + ] + } + , + { + "pattern": "SocialRankIOBot", + "addition_date": "2017/10/19", + "url": "http://socialrank.io/about", + "instances": [ + "SocialRankIOBot; http://socialrank.io/about" + ] + } + , + { + "pattern": "trendictionbot", + "addition_date": "2017/10/30", + "url": "http://www.trendiction.de/bot", + "instances": [ + "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-GB; rv:1.0; trendictionbot0.5.0; trendiction search; http://www.trendiction.de/bot; please let us know of any problems; web at trendiction.com) Gecko/20071127 Firefox/3.0.0.11" + ] + } + , + { + "pattern": "Ocarinabot", + "addition_date": "2017/09/27", + "instances": [ + "Ocarinabot" + ] + } + , + { + "pattern": "epicbot", + "addition_date": "2017/10/31", + "url": "http://www.epictions.com/epicbot", + "instances": [ + "Mozilla/5.0 (compatible; epicbot; +http://www.epictions.com/epicbot)" + ] + } + , + { + "pattern": "Primalbot", + "addition_date": "2017/09/27", + "url": "https://www.primal.com", + "instances": [ + "Mozilla/5.0 (compatible; Primalbot; +https://www.primal.com;)" + ] + } + , + { + "pattern": "DuckDuckGo-Favicons-Bot", + "addition_date": "2017/10/06", + "url": "http://duckduckgo.com", + "instances": [ + "Mozilla/5.0 (compatible; DuckDuckGo-Favicons-Bot/1.0; +http://duckduckgo.com)" + ] + } + , + { + "pattern": "GnowitNewsbot", + "addition_date": "2017/10/30", + "url": "http://www.gnowit.com", + "instances": [ + "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:49.0) Gecko/20100101 Firefox/49.0 / GnowitNewsbot / Contact information at http://www.gnowit.com" + ] + } + , + { + "pattern": "Leikibot", + "addition_date": "2017/09/24", + "url": "http://www.leiki.com", + "instances": [ + "Mozilla/5.0 (Windows NT 6.3;compatible; Leikibot/1.0; +http://www.leiki.com)" + ] + } + , + { + "pattern": "LinkArchiver", + "addition_date": "2017/09/24", + "instances": [ + "@LinkArchiver twitter bot" + ] + } + , + { + "pattern": "YaK\\/", + "addition_date": "2017/09/25", + "url": "http://linkfluence.com", + "instances": [ + "Mozilla/5.0 (compatible; YaK/1.0; http://linkfluence.com/; bot@linkfluence.com)" + ] + } + , + { + "pattern": "PaperLiBot", + "addition_date": "2017/09/25", + "url": "http://support.paper.li/entries/20023257-what-is-paper-li", + "instances": [ + "Mozilla/5.0 (compatible; PaperLiBot/2.1; http://support.paper.li/entries/20023257-what-is-paper-li)", + "Mozilla/5.0 (compatible; PaperLiBot/2.1; https://support.paper.li/entries/20023257-what-is-paper-li)" + + ] + } + , + { + "pattern": "Digg Deeper", + "addition_date": "2017/09/26", + "url": "http://digg.com/about", + "instances": [ + "Digg Deeper/v1 (http://digg.com/about)" + ] + } + , + { + "pattern": "dcrawl", + "addition_date": "2017/09/22", + "instances": [ + "dcrawl/1.0" + ] + } + , + { + "pattern": "Snacktory", + "addition_date": "2017/09/23", + "url": "https://github.com/karussell/snacktory", + "instances": [ + "Mozilla/5.0 (compatible; Snacktory; +https://github.com/karussell/snacktory)" + ] + } + , + { + "pattern": "AndersPinkBot", + "addition_date": "2017/09/24", + "url": "http://anderspink.com/bot.html", + "instances": [ + "Mozilla/5.0 (compatible; AndersPinkBot/1.0; +http://anderspink.com/bot.html)" + ] + } + , + { + "pattern": "Fyrebot", + "addition_date": "2017/09/22", + "instances": [ + "Fyrebot/1.0" + ] + } + , + { + "pattern": "EveryoneSocialBot", + "addition_date": "2017/09/22", + "url": "http://everyonesocial.com", + "instances": [ + "Mozilla/5.0 (compatible; EveryoneSocialBot/1.0; support@everyonesocial.com http://everyonesocial.com/)" + ] + } + , + { + "pattern": "Mediatoolkitbot", + "addition_date": "2017/10/06", + "url": "http://mediatoolkit.com", + "instances": [ + "Mediatoolkitbot (complaints@mediatoolkit.com)" + ] + } + , + { + "pattern": "Luminator-robots", + "addition_date": "2017/09/22", + "instances": [ + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/30.0.1599.66 Safari/537.13 Luminator-robots/2.0" + ] + } + , + { + "pattern": "ExtLinksBot", + "addition_date": "2017/11/02", + "url": "https://extlinks.com/Bot.html", + "instances": [ + "Mozilla/5.0 (compatible; ExtLinksBot/1.5 +https://extlinks.com/Bot.html)" + ] + } + , + { + "pattern": "SurveyBot", + "addition_date": "2017/11/02", + "instances": [ + "Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.9.0.13) Gecko/2009073022 Firefox/3.5.2 (.NET CLR 3.5.30729) SurveyBot/2.3 (DomainTools)" + ] + } + , + { + "pattern": "NING\\/", + "addition_date": "2017/11/02", + "instances": [ + "NING/1.0" + ] + } + , + { + "pattern": "okhttp", + "addition_date": "2017/11/02", + "instances": [ + "okhttp/2.5.0", + "okhttp/2.7.5", + "okhttp/3.2.0", + "okhttp/3.5.0" + ] + } + , + { + "pattern": "Nuzzel", + "addition_date": "2017/11/02", + "instances": [ + "Nuzzel" + ] + } + , + { + "pattern": "omgili", + "addition_date": "2017/11/02", + "url": "http://omgili.com", + "instances": [ + "omgili/0.5 +http://omgili.com" + ] + } + , + { + "pattern": "PocketParser", + "addition_date": "2017/11/02", + "url": "https://getpocket.com/pocketparser_ua", + "instances": [ + "PocketParser/2.0 (+https://getpocket.com/pocketparser_ua)" + ] + } + , + { + "pattern": "YisouSpider", + "addition_date": "2017/11/02", + "instances": [ + "YisouSpider" + ] + } + , + { + "pattern": "um-LN", + "addition_date": "2017/11/02", + "instances": [ + "Mozilla/5.0 (compatible; um-LN/1.0; mailto: techinfo@ubermetrics-technologies.com)" + ] + } + , + { + "pattern": "ToutiaoSpider", + "addition_date": "2017/11/02", + "url": "http://web.toutiao.com/media_cooperation/", + "instances": [ + "Mozilla/5.0 (compatible; ToutiaoSpider/1.0; http://web.toutiao.com/media_cooperation/;)" + ] + } + , + { + "pattern": "MuckRack", + "addition_date": "2017/11/02", + "url": "http://muckrack.com", + "instances": [ + "Mozilla/5.0 (compatible; MuckRack/1.0; +http://muckrack.com)" + ] + } + , + { + "pattern": "Jamie's Spider", + "addition_date": "2017/11/02", + "url": "http://jamiembrown.com/", + "instances": [ + "Jamie's Spider (http://jamiembrown.com/)" + ] + } + , + { + "pattern": "AHC\\/", + "addition_date": "2017/11/02", + "instances": [ + "AHC/2.0" + ] + } + , + { + "pattern": "NetcraftSurveyAgent", + "addition_date": "2017/11/02", + "instances": [ + "Mozilla/5.0 (compatible; NetcraftSurveyAgent/1.0; +info@netcraft.com)" + ] + } + , + { + "pattern": "Laserlikebot", + "addition_date": "2017/11/02", + "instances": [ + "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Laserlikebot/0.1)" + ] + } + , + { + "pattern": "Apache-HttpClient", + "addition_date": "2017/11/02", + "instances": [ + "Apache-HttpClient/4.2.3 (java 1.5)", + "Apache-HttpClient/4.2.5 (java 1.5)", + "Apache-HttpClient/4.3.1 (java 1.5)", + "Apache-HttpClient/4.3.3 (java 1.5)", + "Apache-HttpClient/4.3.5 (java 1.5)", + "Apache-HttpClient/4.4.1 (Java/1.8.0_65)", + "Apache-HttpClient/4.5.3 (Java/1.8.0_121)" + ] + } + , + { + "pattern": "AppEngine-Google", + "addition_date": "2017/11/02", + "instances": [ + "AppEngine-Google; (+http://code.google.com/appengine; appid: example)" + ] + } + , + { + "pattern": "Jetty", + "addition_date": "2017/11/02", + "instances": [ + "Jetty/9.3.z-SNAPSHOT" + ] + } + , + { + "pattern": "Upflow", + "addition_date": "2017/11/02", + "instances": [ + "Upflow/1.0" + ] + } + , + { + "pattern": "Thinklab", + "addition_date": "2017/11/02", + "url": "thinklab.com", + "instances": [ + "Thinklab (thinklab.com)" + ] + } + , + { + "pattern": "Traackr.com", + "addition_date": "2017/11/02", + "url": "Traackr.com", + "instances": [ + "Traackr.com" + ] + } + , + { + "pattern": "Twurly", + "addition_date": "2017/11/02", + "url": "http://twurly.org", + "instances": [ + "Ruby, Twurly v1.1 (http://twurly.org)" + ] + } + , + { + "pattern": "Mastodon", + "addition_date": "2017/11/02", + "instances": [ + "http.rb/2.2.2 (Mastodon/1.5.1; +https://example-masto-instance.org/)" + ] + } + , + { + "pattern": "http_get", + "addition_date": "2017/11/02", + "instances": [ + "http_get" + ] + } + , + { + "pattern": "DnyzBot", + "addition_date": "2017/11/20", + "instances": [ + "Mozilla/5.0 (compatible; DnyzBot/1.0)", + "Mozilla/5.0 (compatible; DnyzBot/1.0) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/64.0.3282.167 Safari/537.36", + "Mozilla/5.0 (compatible; DnyzBot/1.0) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/64.0.3264.0 Safari/537.36" + ] + } + , + { + "pattern": "botify", + "addition_date": "2018/02/01", + "instances": [ + "Mozilla/5.0 (compatible; botify; http://botify.com)" + ] + } + , + { + "pattern": "007ac9 Crawler", + "addition_date": "2018/02/09", + "instances": [ + "Mozilla/5.0 (compatible; 007ac9 Crawler; http://crawler.007ac9.net/)" + ] + } + , + { + "pattern": "BehloolBot", + "addition_date": "2018/02/09", + "instances": [ + "Mozilla/5.0 (compatible; BehloolBot/beta; +http://www.webeaver.com/bot)" + ] + } + , + { + "pattern": "BrandVerity", + "addition_date": "2018/02/27", + "instances": [ + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:41.0) Gecko/20100101 Firefox/55.0 BrandVerity/1.0 (http://www.brandverity.com/why-is-brandverity-visiting-me)" + ] + } + , + { + "pattern": "check_http", + "addition_date": "2018/02/09", + "instances": [ + "check_http/v2.2.1 (nagios-plugins 2.2.1)" + ] + } + , + { + "pattern": "BDCbot", + "addition_date": "2018/02/09", + "instances": [ + "Mozilla/5.0 (Windows NT 6.1; compatible; BDCbot/1.0; +http://bigweb.bigdatacorp.com.br/faq.aspx) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Safari/537.36" + ] + } + , + { + "pattern": "ZumBot", + "addition_date": "2018/02/09", + "instances": [ + "Mozilla/5.0 (compatible; ZumBot/1.0; http://help.zum.com/inquiry)" + ] + } + , + { + "pattern": "EZID", + "addition_date": "2018/02/09", + "instances": [ + "EZID (EZID link checker; https://ezid.cdlib.org/)" + ] + } + , + { + "pattern": "ICC-Crawler", + "addition_date": "2018/02/28", + "instances": [ + "ICC-Crawler/2.0 (Mozilla-compatible; ; http://ucri.nict.go.jp/en/icccrawler.html)" + ], + "url": "http://ucri.nict.go.jp/en/icccrawler.html" + } + , + { + "pattern": "ArchiveBot", + "addition_date": "2018/02/28", + "instances": [ + "ArchiveTeam ArchiveBot/20170106.02 (wpull 2.0.2)" + ], + "url": "https://github.com/ArchiveTeam/ArchiveBot" + } + , + { + "pattern": "^LCC ", + "addition_date": "2018/02/28", + "instances": [ + "LCC (+http://corpora.informatik.uni-leipzig.de/crawler_faq.html)" + ], + "url": "http://corpora.informatik.uni-leipzig.de/crawler_faq.html" + } + , + { + "pattern": "filterdb.iss.net\\/crawler", + "addition_date": "2018/03/16", + "instances": [ + "Mozilla/5.0 (compatible; oBot/2.3.1; +http://filterdb.iss.net/crawler/)" + ], + "url": "http://filterdb.iss.net/crawler/" + } + , + { + "pattern": "BLP_bbot", + "addition_date": "2018/03/27", + "instances": [ + "BLP_bbot/0.1" + ] + } + , + { + "pattern": "BomboraBot", + "addition_date": "2018/03/27", + "instances": [ + "Mozilla/5.0 (compatible; BomboraBot/1.0; +http://www.bombora.com/bot)" + ], + "url": "http://www.bombora.com/bot" + } + , + { + "pattern": "Buck\\/", + "addition_date": "2018/03/27", + "instances": [ + "Buck/2.2; (+https://app.hypefactors.com/media-monitoring/about.html)" + ], + "url": "https://app.hypefactors.com/media-monitoring/about.html" + } + , + { + "pattern": "Companybook-Crawler", + "addition_date": "2018/03/27", + "instances": [ + "Companybook-Crawler (+https://www.companybooknetworking.com/)" + ], + "url": "https://www.companybooknetworking.com/" + } + , + { + "pattern": "Genieo", + "addition_date": "2018/03/27", + "instances": [ + "Mozilla/5.0 (compatible; Genieo/1.0 http://www.genieo.com/webfilter.html)" + ], + "url": "http://www.genieo.com/webfilter.html" + } + , + { + "pattern": "magpie-crawler", + "addition_date": "2018/03/27", + "instances": [ + "magpie-crawler/1.1 (U; Linux amd64; en-GB; +http://www.brandwatch.net)" + ], + "url": "http://www.brandwatch.net" + } + , + { + "pattern": "MeltwaterNews", + "addition_date": "2018/03/27", + "instances": [ + "MeltwaterNews www.meltwater.com" + ], + "url": "http://www.meltwater.com" + } + , + { + "pattern": "Moreover", + "addition_date": "2018/03/27", + "instances": [ + "Mozilla/5.0 Moreover/5.1 (+http://www.moreover.com)" + ], + "url": "http://www.moreover.com" + } + , + { + "pattern": "newspaper\\/", + "addition_date": "2018/03/27", + "instances": [ + "newspaper/0.2.5", + "newspaper/0.2.6", + "newspaper/0.1.0.7" + ] + } + , + { + "pattern": "ScoutJet", + "addition_date": "2018/03/27", + "instances": [ + "Mozilla/5.0 (compatible; ScoutJet; +http://www.scoutjet.com/)" + ], + "url": "http://www.scoutjet.com/" + } + , + { + "pattern": "(^| )sentry\\/", + "addition_date": "2018/03/27", + "instances": [ + "sentry/8.22.0 (https://sentry.io)" + ], + "url": "https://sentry.io" + } + , + { + "pattern": "StorygizeBot", + "addition_date": "2018/03/27", + "instances": [ + "Mozilla/5.0 (compatible; StorygizeBot; http://www.storygize.com)" + ], + "url": "http://www.storygize.com" + } + , + { + "pattern": "UptimeRobot", + "addition_date": "2018/03/27", + "instances": [ + "Mozilla/5.0+(compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)" + ], + "url": "http://www.uptimerobot.com/" + } + , + { + "pattern": "OutclicksBot", + "addition_date": "2018/04/21", + "instances": [ + "OutclicksBot/2 +https://www.outclicks.net/agent/VjzDygCuk4ubNmg40ZMbFqT0sIh7UfOKk8s8ZMiupUR", + "OutclicksBot/2 +https://www.outclicks.net/agent/gIYbZ38dfAuhZkrFVl7sJBFOUhOVct6J1SvxgmBZgCe", + "OutclicksBot/2 +https://www.outclicks.net/agent/PryJzTl8POCRHfvEUlRN5FKtZoWDQOBEvFJ2wh6KH5J", + "OutclicksBot/2 +https://www.outclicks.net/agent/p2i4sNUh7eylJF1S6SGgRs5mP40ExlYvsr9GBxVQG6h" + ], + "url": "https://www.outclicks.net" + } + , + { + "pattern": "seoscanners", + "addition_date": "2018/05/27", + "instances": [ + "Mozilla/5.0 (compatible; seoscanners.net/1; +spider@seoscanners.net)" + ], + "url": "http://www.seoscanners.net/" + } + , + { + "pattern": "Hatena", + "addition_date": "2018/05/29", + "instances": [ + "Hatena Antenna/0.3", + "Hatena::Russia::Crawler/0.01" + ] + } + , + { + "pattern": "Google Web Preview", + "addition_date": "2018/05/31", + "instances": [ + "Mozilla/5.0 (Linux; U; Android 2.3.4; generic) AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview) Version/4.0 Mobile Safari/537.36" + ] + } + , + { + "pattern": "MauiBot", + "addition_date": "2018/06/06", + "instances": [ + "MauiBot (crawler.feedback+wc@gmail.com)" + ] + } + , + { + "pattern": "AlphaBot", + "addition_date": "2018/05/27", + "instances": [ + "Mozilla/5.0 (compatible; AlphaBot/3.2; +http://alphaseobot.com/bot.html)" + ], + "url": "http://alphaseobot.com/bot.html" + } + , + { + "pattern": "SBL-BOT", + "addition_date": "2018/06/06", + "instances": [ + "SBL-BOT (http://sbl.net)" + ], + "url": "http://sbl.net", + "description" : "Bot of SoftByte BlackWidow" + } + , + { + "pattern": "IAS crawler", + "addition_date": "2018/06/06", + "instances": [ + "IAS crawler (ias_crawler; http://integralads.com/site-indexing-policy/)" + ], + "url": "http://integralads.com/site-indexing-policy/", + "description" : "Bot of Integral Ad Science, Inc." + } + , + { + "pattern": "adscanner", + "addition_date": "2018/06/24", + "instances": [ + "Mozilla/5.0 (compatible; adscanner/)" + ] + } + , + { + "pattern": "Netvibes", + "addition_date": "2018/06/24", + "instances": [ + "Netvibes (crawler/bot; http://www.netvibes.com", + "Netvibes (crawler; http://www.netvibes.com)" + ], + "url": "http://www.netvibes.com" + } + , + { + "pattern": "acapbot", + "addition_date": "2018/06/27", + "instances": [ + "Mozilla/5.0 (compatible;acapbot/0.1;treat like Googlebot)", + "Mozilla/5.0 (compatible;acapbot/0.1.;treat like Googlebot)" + ] + } + , + { + "pattern": "Baidu-YunGuanCe", + "addition_date": "2018/06/27", + "instances": [ + "Baidu-YunGuanCe-Bot(ce.baidu.com)", + "Baidu-YunGuanCe-SLABot(ce.baidu.com)", + "Baidu-YunGuanCe-ScanBot(ce.baidu.com)", + "Baidu-YunGuanCe-PerfBot(ce.baidu.com)", + "Baidu-YunGuanCe-VSBot(ce.baidu.com)" + ], + "url": "https://ce.baidu.com/topic/topic20150908", + "description": "Baidu Cloud Watch" + } + , + { + "pattern": "bitlybot", + "addition_date": "2018/06/27", + "instances": [ + "bitlybot/3.0 (+http://bit.ly/)", + "bitlybot/2.0", + "bitlybot" + ], + "url": "http://bit.ly/" + } + , + { + "pattern": "blogmuraBot", + "addition_date": "2018/06/27", + "instances": [ + "blogmuraBot (+http://www.blogmura.com)" + ], + "url": "http://www.blogmura.com", + "description": "A blog ranking site which links to blogs on just about every theme possible." + } + , + { + "pattern": "Bot.AraTurka.com", + "addition_date": "2018/06/27", + "instances": [ + "Bot.AraTurka.com/0.0.1" + ], + "url": "http://www.araturka.com" + } + , + { + "pattern": "bot-pge.chlooe.com", + "addition_date": "2018/06/27", + "instances": [ + "bot-pge.chlooe.com/1.0.0 (+http://www.chlooe.com/)" + ] + } + , + { + "pattern": "BoxcarBot", + "addition_date": "2018/06/27", + "instances": [ + "Mozilla/5.0 (compatible; BoxcarBot/1.1; +awesome@boxcar.io)" + ], + "url": "https://boxcar.io/" + } + , + { + "pattern": "BTWebClient", + "addition_date": "2018/06/27", + "instances": [ + "BTWebClient/180B(9704)" + ], + "url": "http://www.utorrent.com/", + "description": "µTorrent BitTorrent Client" + } + , + { + "pattern": "ContextAd Bot", + "addition_date": "2018/06/27", + "instances": [ + "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0;.NET CLR 1.0.3705; ContextAd Bot 1.0)", + "ContextAd Bot 1.0" + ] + } + , + { + "pattern": "Digincore bot", + "addition_date": "2018/06/27", + "instances": [ + "Mozilla/5.0 (compatible; Digincore bot; https://www.digincore.com/crawler.html for rules and instructions.)" + ], + "url": "http://www.digincore.com/crawler.html" + } + , + { + "pattern": "Disqus", + "addition_date": "2018/06/27", + "instances": [ + "Disqus/1.0" + ], + "url": "https://disqus.com/", + "description": "validate and quality check pages." + } + , + { + "pattern": "Feedly", + "addition_date": "2018/06/27", + "instances": [ + "Feedly/1.0 (+http://www.feedly.com/fetcher.html; like FeedFetcher-Google)", + "FeedlyBot/1.0 (http://feedly.com)" + ], + "url": "https://www.feedly.com/fetcher.html", + "description": "Feedly Fetcher is how Feedly grabs RSS or Atom feeds when users choose to add them to their Feedly or any of the other applications built on top of the feedly cloud." + } + , + { + "pattern": "Fetch\\/", + "addition_date": "2018/06/27", + "instances": [ + "Fetch/2.0a (CMS Detection/Web/SEO analysis tool, see http://guess.scritch.org)" + ] + } + , + { + "pattern": "Fever", + "addition_date": "2018/06/27", + "instances": [ + "Fever/1.38 (Feed Parser; http://feedafever.com; Allow like Gecko)" + ], + "url": "http://feedafever.com" + } + , + { + "pattern": "Flamingo_SearchEngine", + "addition_date": "2018/06/27", + "instances": [ + "Flamingo_SearchEngine (+http://www.flamingosearch.com/bot)" + ] + } + , + { + "pattern": "FlipboardProxy", + "addition_date": "2018/06/27", + "instances": [ + "Mozilla/5.0 (compatible; FlipboardProxy/1.1; +http://flipboard.com/browserproxy)", + "Mozilla/5.0 (compatible; FlipboardProxy/1.2; +http://flipboard.com/browserproxy)", + "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6 (FlipboardProxy/1.1; +http://flipboard.com/browserproxy)", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:28.0) Gecko/20100101 Firefox/28.0 (FlipboardProxy/1.1; +http://flipboard.com/browserproxy)" + ], + "url": "https://about.flipboard.com/browserproxy/", + "description": "a proxy service to fetch, validate, and prepare certain elements of websites for presentation through the Flipboard Application" + } + , + { + "pattern": "g2reader-bot", + "addition_date": "2018/06/27", + "instances": [ + "g2reader-bot/1.0 (+http://www.g2reader.com/)" + ], + "url": "http://www.g2reader.com/" + } + , + { + "pattern": "imrbot", + "addition_date": "2018/06/27", + "instances": [ + "Mozilla/5.0 (compatible; imrbot/1.10.8 +http://www.mignify.com)" + ], + "url": "http://www.mignify.com" + } + , + { + "pattern": "K7MLWCBot", + "addition_date": "2018/06/27", + "instances": [ + "K7MLWCBot/1.0 (+http://www.k7computing.com)" + ], + "url": "http://www.k7computing.com", + "description": "Virus scanner" + } + , + { + "pattern": "Kemvibot", + "addition_date": "2018/06/27", + "instances": [ + "Kemvibot/1.0 (http://kemvi.com, marco@kemvi.com)" + ], + "url": "http://kemvi.com" + } + , + { + "pattern": "Landau-Media-Spider", + "addition_date": "2018/06/27", + "instances": [ + "Landau-Media-Spider/1.0(http://bots.landaumedia.de/bot.html)" + ], + "url": "http://bots.landaumedia.de/bot.html" + } + , + { + "pattern": "linkapediabot", + "addition_date": "2018/06/27", + "instances": [ + "linkapediabot (+http://www.linkapedia.com)" + ], + "url": "http://www.linkapedia.com" + } + , + { + "pattern": "vkShare", + "addition_date": "2018/07/02", + "instances": [ + "Mozilla/5.0 (compatible; vkShare; +http://vk.com/dev/Share)" + ], + "url": "http://vk.com/dev/Share" + } + , + { + "pattern": "Siteimprove.com", + "addition_date": "2018/06/22", + "instances": [ + "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) LinkCheck by Siteimprove.com", + "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.0) Match by Siteimprove.com", + "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) SiteCheck-sitecrawl by Siteimprove.com", + "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.0) LinkCheck by Siteimprove.com" + ] + } + , + { + "pattern": "BLEXBot\\/", + "addition_date": "2018/07/07", + "instances": [ + "Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)" + ], + "url": "http://webmeup-crawler.com" + } + , + { + "pattern": "DareBoost", + "addition_date": "2018/07/07", + "instances": [ + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.75 Safari/537.36 DareBoost" + ], + "url": "https://www.dareboost.com/", + "description": "Bot to test, Analyze and Optimize website" + } + , + { + "pattern": "ZuperlistBot\\/", + "addition_date": "2018/07/07", + "instances": [ + "Mozilla/5.0 (compatible; ZuperlistBot/1.0)" + ] + } + , + { + "pattern": "Miniflux\\/", + "addition_date": "2018/07/07", + "instances": [ + "Mozilla/5.0 (compatible; Miniflux/2.0.x-dev; +https://miniflux.net)", + "Mozilla/5.0 (compatible; Miniflux/2.0.3; +https://miniflux.net)", + "Mozilla/5.0 (compatible; Miniflux/2.0.7; +https://miniflux.net)", + "Mozilla/5.0 (compatible; Miniflux/2.0.10; +https://miniflux.net)", + "Mozilla/5.0 (compatibl$; Miniflux/2.0.x-dev; +https://miniflux.app)", + "Mozilla/5.0 (compatible; Miniflux/2.0.11; +https://miniflux.app)", + "Mozilla/5.0 (compatible; Miniflux/2.0.12; +https://miniflux.app)", + "Mozilla/5.0 (compatible; Miniflux/ae1dc1a; +https://miniflux.app)", + "Mozilla/5.0 (compatible; Miniflux/3b6e44c; +https://miniflux.app)" + ], + "url": "https://miniflux.net", + "description": "Miniflux is a minimalist and opinionated feed reader." + } + , + { + "pattern": "Feedspot", + "addition_date": "2018/07/07", + "instances": [ + "Mozilla/5.0 (compatible; Feedspotbot/1.0; +http://www.feedspot.com/fs/bot)", + "Mozilla/5.0 (compatible; Feedspot/1.0 (+https://www.feedspot.com/fs/fetcher; like FeedFetcher-Google)" + ], + "url": "http://www.feedspot.com/fs/bot" + } + , + { + "pattern": "Diffbot\\/", + "addition_date": "2018/07/07", + "instances": [ + "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.2) Gecko/20090729 Firefox/3.5.2 (.NET CLR 3.5.30729; Diffbot/0.1; +http://www.diffbot.com)" + ], + "url": "http://www.diffbot.com" + } + , + { + "pattern": "SEOkicks", + "addition_date": "2018/08/22", + "instances": [ + "Mozilla/5.0 (compatible; SEOkicks; +https://www.seokicks.de/robot.html)" + ], + "url": "https://www.seokicks.de/robot.html" + } + , + { + "pattern": "tracemyfile", + "addition_date": "2018/08/23", + "instances": [ + "Mozilla/5.0 (compatible; tracemyfile/1.0; +bot@tracemyfile.com)" + ] + } + , + { + "pattern": "Nimbostratus-Bot", + "addition_date": "2018/08/29", + "instances": [ + "Mozilla/5.0 (compatible; Nimbostratus-Bot/v1.3.2; http://cloudsystemnetworks.com)" + ] + } + , + { + "pattern": "zgrab", + "addition_date": "2018/08/30", + "instances": [ + "Mozilla/5.0 zgrab/0.x" + ], + "url": "https://zmap.io/" + } + , + { + "pattern": "PR-CY.RU", + "addition_date": "2018/08/30", + "instances": [ + "Mozilla/5.0 (compatible; PR-CY.RU; + https://a.pr-cy.ru)" + ], + "url": "https://a.pr-cy.ru/" + } + , + { + "pattern": "AdsTxtCrawler", + "addition_date": "2018/08/30", + "instances": [ + "AdsTxtCrawler/1.0" + ] + }, + { + "pattern": "Datafeedwatch", + "addition_date": "2018/09/05", + "instances": [ + "Datafeedwatch/2.1.x" + ], + "url": "https://www.datafeedwatch.com/" + } + , + { + "pattern": "Zabbix", + "addition_date": "2018/09/05", + "instances": [ + "Zabbix" + ], + "url": "https://www.zabbix.com/documentation/3.4/manual/web_monitoring" + } + , + { + "pattern": "TangibleeBot", + "addition_date": "2018/09/05", + "instances": [ + "TangibleeBot/1.0.0.0 (http://tangiblee.com/bot)" + ], + "url": "http://tangiblee.com/bot" + } + , + { + "pattern": "google-xrawler", + "addition_date": "2018/09/05", + "instances": [ + "google-xrawler" + ], + "url": "https://webmasters.stackexchange.com/questions/105560/what-is-the-google-xrawler-user-agent-used-for" + } + , + { + "pattern": "axios", + "addition_date": "2018/09/06", + "instances": [ + "axios/0.18.0" + ], + "url": "https://github.com/axios/axios" + } + , + { + "pattern": "Amazon CloudFront", + "addition_date": "2018/09/07", + "instances": [ + "Amazon CloudFront" + ], + "url": "https://aws.amazon.com/cloudfront/" + } + , + { + "pattern": "Pulsepoint", + "addition_date": "2018/09/24", + "instances": [ + "Pulsepoint XT3 web scraper" + ] + } + , + { + "pattern": "CloudFlare-AlwaysOnline", + "addition_date": "2018/09/27", + "instances": [ + "Mozilla/5.0 (compatible; CloudFlare-AlwaysOnline/1.0; +http://www.cloudflare.com/always-online) AppleWebKit/534.34", + "Mozilla/5.0 (compatible; CloudFlare-AlwaysOnline/1.0; +https://www.cloudflare.com/always-online) AppleWebKit/534.34" + ], + "url" : "https://www.cloudflare.com/always-online/" + } + , + { + "pattern": "Google-Structured-Data-Testing-Tool", + "addition_date": "2018/10/02", + "instances": [ + "Mozilla/5.0 (compatible; Google-Structured-Data-Testing-Tool +https://search.google.com/structured-data/testing-tool)", + "Mozilla/5.0 (compatible; Google-Structured-Data-Testing-Tool +http://developers.google.com/structured-data/testing-tool/)" + ], + "url": "https://search.google.com/structured-data/testing-tool" + } + , + { + "pattern": "WordupInfoSearch", + "addition_date": "2018/10/07", + "instances": [ + "WordupInfoSearch/1.0" + ] + } + , + { + "pattern": "WebDataStats", + "addition_date": "2018/10/08", + "instances": [ + "Mozilla/5.0 (compatible; WebDataStats/1.0 ; +https://webdatastats.com/policy.html)" + ], + "url": "https://webdatastats.com/" + } + , + { + "pattern": "HttpUrlConnection", + "addition_date": "2018/10/08", + "instances": [ + "Jersey/2.25.1 (HttpUrlConnection 1.8.0_141)" + ] + } + , + { + "pattern": "Seekport Crawler", + "addition_date": "2018/10/08", + "instances": [ + "Mozilla/5.0 (compatible; Seekport Crawler; http://seekport.com/)" + ], + "url": "http://seekport.com/" + } + , + { + "pattern": "ZoomBot", + "addition_date": "2018/10/10", + "instances": [ + "ZoomBot (Linkbot 1.0 http://suite.seozoom.it/bot.html)" + ], + "url": "http://suite.seozoom.it/bot.html" + } + , + { + "pattern": "VelenPublicWebCrawler", + "addition_date": "2018/10/09", + "instances": [ + "VelenPublicWebCrawler (velen.io)" + ] + } + , + { + "pattern": "MoodleBot", + "addition_date": "2018/10/10", + "instances": [ + "MoodleBot/1.0" + ] + } + , + { + "pattern": "jpg-newsbot", + "addition_date": "2018/10/10", + "instances": [ + "jpg-newsbot/2.0; (+https://vipnytt.no/bots/)" + ], + "url": "https://vipnytt.no/bots/" + } + , + { + "pattern": "outbrain", + "addition_date": "2018/10/14", + "instances": [ + "Mozilla/5.0 (Java) outbrain" + ], + "url": "https://www.outbrain.com/help/advertisers/invalid-url/" + } + , + { + "pattern": "W3C_Validator", + "addition_date": "2018/10/14", + "instances": [ + "W3C_Validator/1.3" + ], + "url": "https://validator.w3.org/services" + } + , + { + "pattern": "Validator\\.nu", + "addition_date": "2018/10/14", + "instances": [ + "Validator.nu/LV" + ], + "url": "https://validator.w3.org/services" + } + , + { + "pattern": "W3C-checklink", + "addition_date": "2018/10/14", + "instances": [ + "W3C-checklink" + ], + "url": "https://validator.w3.org/services" + } + , + { + "pattern": "W3C-mobileOK", + "addition_date": "2018/10/14", + "instances": [ + "W3C-mobileOK/DDC-1.0" + ], + "url": "https://validator.w3.org/services" + } + , + { + "pattern": "W3C_I18n-Checker", + "addition_date": "2018/10/14", + "instances": [ + "W3C_I18n-Checker/1.0" + ], + "url": "https://validator.w3.org/services" + } + , + { + "pattern": "FeedValidator", + "addition_date": "2018/10/14", + "instances": [ + "FeedValidator/1.3" + ], + "url": "https://validator.w3.org/services" + } + , + { + "pattern": "W3C_CSS_Validator", + "addition_date": "2018/10/14", + "instances": [ + "Jigsaw/2.3.0 W3C_CSS_Validator_JFouffa/2.0" + ], + "url": "https://validator.w3.org/services" + } + , + { + "pattern": "W3C_Unicorn", + "addition_date": "2018/10/14", + "instances": [ + "W3C_Unicorn/1.0" + ], + "url": "https://validator.w3.org/services" + } + , + { + "pattern": "Google-PhysicalWeb", + "addition_date": "2018/10/21", + "instances": [ + "Mozilla/5.0 (Google-PhysicalWeb)" + ] + } + , + { + "pattern": "Blackboard", + "addition_date": "2018/10/28", + "instances": [ + "Blackboard Safeassign" + ], + "url": "https://help.blackboard.com/Learn/Administrator/Hosting/Tools_Management/SafeAssign" + }, + { + "pattern": "ICBot\\/", + "addition_date": "2018/10/23", + "instances": [ + "Mozilla/5.0 (compatible; ICBot/0.1; +https://ideasandcode.xyz" + ], + "url": "https://ideasandcode.xyz" + }, + { + "pattern": "BazQux", + "addition_date": "2018/10/23", + "instances": [ + "Mozilla/5.0 (compatible; BazQux/2.4; +https://bazqux.com/fetcher; 1 subscribers)" + ], + "url": "https://bazqux.com/fetcher" + }, + { + "pattern": "Twingly", + "addition_date": "2018/10/23", + "instances": [ + "Mozilla/5.0 (compatible; Twingly Recon; twingly.com)" + ], + "url": "https://twingly.com" + }, + { + "pattern": "Rivva", + "addition_date": "2018/10/23", + "instances": [ + "Mozilla/5.0 (compatible; Rivva; http://rivva.de)" + ], + "url": "http://rivva.de" + }, + { + "pattern": "Experibot", + "addition_date": "2018/11/03", + "instances": [ + "Experibot-v2 http://goo.gl/ZAr8wX", + "Experibot-v3 http://goo.gl/ZAr8wX" + ], + "url": "https://amirkr.wixsite.com/experibot" + }, + { + "pattern": "awesomecrawler", + "addition_date": "2018/11/24", + "instances": [ + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.22 (KHTML, like Gecko) Chrome/25.0.1364.5 Safari/537.22 +awesomecrawler" + ] + }, + { + "pattern": "Dataprovider.com", + "addition_date": "2018/11/24", + "instances": [ + "Mozilla/5.0 (compatible; Dataprovider.com)" + ], + "url": "https://www.dataprovider.com/" + }, + { + "pattern": "GroupHigh\\/", + "addition_date": "2018/11/24", + "instances": [ + "Mozilla/5.0 (compatible; GroupHigh/1.0; +http://www.grouphigh.com/" + ], + "url": "http://www.grouphigh.com/" + }, + { + "pattern": "theoldreader.com", + "addition_date": "2018/12/02", + "instances": [ + "Mozilla/5.0 (compatible; theoldreader.com)" + ], + "url": "https://www.theoldreader.com/" + } + , + { + "pattern": "AnyEvent", + "addition_date": "2018/12/07", + "instances": [ + "Mozilla/5.0 (compatible; U; AnyEvent-HTTP/2.24; +http://software.schmorp.de/pkg/AnyEvent)" + ], + "url": "http://software.schmorp.de/pkg/AnyEvent.html" + } + , + { + "pattern": "Uptimebot", + "addition_date": "2019/01/17", + "instances": [ + "Uptimebot.org - Free website monitoring" + ], + "url": "http://uptimebot.org/" + } + , + { + "pattern": "Nmap Scripting Engine", + "addition_date": "2019/02/04", + "instances": [ + "Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)" + ], + "url": "https://nmap.org/book/nse.html" + } +] diff --git a/lib/impressionist/bots.rb b/lib/impressionist/bots.rb index 93c11d2d..d6da72a4 100644 --- a/lib/impressionist/bots.rb +++ b/lib/impressionist/bots.rb @@ -1,20 +1,19 @@ require 'timeout' require 'net/http' -require 'nokogiri' +require 'json' module Impressionist module Bots - LIST_URL = "http://www.user-agents.org/allagents.xml" + LIST_URL = "https://raw.githubusercontent.com/monperrus/crawler-user-agents/master/crawler-user-agents.json" def self.consume Timeout.timeout(4) do response = Net::HTTP.get(URI.parse(LIST_URL)) - doc = Nokogiri::XML(response) + my_hash = JSON.parse(response) list = [] - doc.xpath('//user-agent').each do |agent| - type = agent.xpath("Type").text - list << agent.xpath("String").text.gsub("<","<") if ["R","S"].include?(type) #gsub hack for badly formatted data + my_hash.each do |agent| + list << agent['instances'] end - list + pp list.flatten end end end diff --git a/tests/test_app/spec/models/bots_spec.rb b/tests/test_app/spec/models/bots_spec.rb index a2983587..ee347311 100644 --- a/tests/test_app/spec/models/bots_spec.rb +++ b/tests/test_app/spec/models/bots_spec.rb @@ -8,7 +8,7 @@ end it "is true if user_agent is on bot list" do - Impressionist::Bots.bot?("A-Online Search").should be_true + Impressionist::Bots.bot?("Domain Re-Animator Bot (http://domainreanimator.com) - support@domainreanimator.com").should be_true end it "is false if user_agent is blank" do @@ -24,4 +24,4 @@ Impressionist::Bots.bot?.should be_false end end -end \ No newline at end of file +end