How to use Pattern method of html Package

Best K6 code snippet using html.Pattern

crawler-user-agents.go

Source:crawler-user-agents.go

2import (3 "encoding/json"4 "sync"5)6type CrawlerPattern struct {7 Pattern string `json:"pattern"`8 AdditionDate string `json:"addition_date"`9 Instances []string `json:"instances"`10 Url string `json:"url"`11}12var parseOnce sync.Once13var crawlerPatterns []CrawlerPattern14func getCrawlerPatterns() []CrawlerPattern {15 parseOnce.Do(func() {16 crawlerPatterns = parseCrawlerPatterns()17 })18 return crawlerPatterns19}20func parseCrawlerPatterns() []CrawlerPattern {21 crawlers := `22 [23 {24 "pattern": "Googlebot\\/",25 "url": "http://www.google.com/bot.html",26 "instances": [27 "Googlebot/2.1 (+http://www.google.com/bot.html)",28 "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",29 "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",30 "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",31 "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",32 "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",33 "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",34 "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Safari/537.36"35 ]36 }37 ,38 {39 "pattern": "Googlebot-Mobile",40 "instances": [41 "DoCoMo/2.0 N905i(c100;TB;W24H16) (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)",42 "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)",43 "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_1 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8B117 Safari/6531.22.7 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)",44 "Nokia6820/2.0 (4.83) Profile/MIDP-1.0 Configuration/CLDC-1.0 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)",45 "SAMSUNG-SGH-E250/1.0 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Browser/6.2.3.3.c.1.101 (GUI) MMP/2.0 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)"46 ]47 }48 ,49 {50 "pattern": "Googlebot-Image",51 "instances": [52 "Googlebot-Image/1.0"53 ]54 }55 ,56 {57 "pattern": "Googlebot-News",58 "instances": [59 "Googlebot-News"60 ]61 }62 ,63 {64 "pattern": "Googlebot-Video",65 "instances": [66 "Googlebot-Video/1.0"67 ]68 }69 ,70 {71 "pattern": "AdsBot-Google([^-]|$)",72 "url": "https://support.google.com/webmasters/answer/1061943?hl=en",73 "instances": [74 "AdsBot-Google (+http://www.google.com/adsbot.html)"75 ]76 }77 ,78 {79 "pattern": "AdsBot-Google-Mobile",80 "addition_date": "2017/08/21",81 "url": "https://support.google.com/adwords/answer/2404197",82 "instances": [83 "AdsBot-Google-Mobile-Apps",84 "Mozilla/5.0 (Linux; Android 5.0; SM-G920A) AppleWebKit (KHTML, like Gecko) Chrome Mobile Safari (compatible; AdsBot-Google-Mobile; +http://www.google.com/mobile/adsbot.html)",85 "Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1 (compatible; AdsBot-Google-Mobile; +http://www.google.com/mobile/adsbot.html)"86 ]87 }88 ,89 {90 "pattern": "Feedfetcher-Google",91 "addition_date": "2018/06/27",92 "url": "https://support.google.com/webmasters/answer/178852",93 "instances": [94 "Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 1 subscribers; feed-id=728742641706423)"95 ]96 }97 ,98 {99 "pattern": "Mediapartners-Google",100 "url": "https://support.google.com/webmasters/answer/1061943?hl=en",101 "instances": [102 "Mediapartners-Google",103 "Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;) Daumoa/4.0 (Following Mediapartners-Google)",104 "Mozilla/5.0 (iPhone; U; CPU iPhone OS 10_0 like Mac OS X; en-us) AppleWebKit/602.1.38 (KHTML, like Gecko) Version/10.0 Mobile/14A5297c Safari/602.1 (compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)",105 "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_1 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8B117 Safari/6531.22.7 (compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)"106 ]107 }108 ,109 {110 "pattern": "Mediapartners \\(Googlebot\\)",111 "addition_date": "2017/08/08",112 "url": "https://support.google.com/webmasters/answer/1061943?hl=en",113 "instances": []114 }115 ,116 {117 "pattern": "APIs-Google",118 "addition_date": "2017/08/08",119 "url": "https://support.google.com/webmasters/answer/1061943?hl=en",120 "instances": [121 "APIs-Google (+https://developers.google.com/webmasters/APIs-Google.html)"122 ]123 }124 ,125 {126 "pattern": "bingbot",127 "url": "http://www.bing.com/bingbot.htm",128 "instances": [129 "Mozilla/5.0 (Windows Phone 8.1; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 530) like Gecko (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)",130 "Mozilla/5.0 (compatible; adidxbot/2.0; http://www.bing.com/bingbot.htm)",131 "Mozilla/5.0 (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)",132 "Mozilla/5.0 (compatible; bingbot/2.0; http://www.bing.com/bingbot.htm)",133 "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm",134 "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)",135 "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) SitemapProbe",136 "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; adidxbot/2.0; http://www.bing.com/bingbot.htm)",137 "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)",138 "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; http://www.bing.com/bingbot.htm)",139 "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)",140 "Mozilla/5.0 (seoanalyzer; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)",141 "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Safari/537.36"142 ]143 }144 ,145 {146 "pattern": "Slurp",147 "url": "http://help.yahoo.com/help/us/ysearch/slurp",148 "instances": [149 "Mozilla/5.0 (compatible; Yahoo! Slurp/3.0; http://help.yahoo.com/help/us/ysearch/slurp)",150 "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)",151 "Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)"152 ]153 }154 ,155 {156 "pattern": "[wW]get",157 "instances": [158 "WGETbot/1.0 (+http://wget.alanreed.org)",159 "Wget/1.14 (linux-gnu)",160 "Wget/1.20.3 (linux-gnu)"161 ]162 }163 ,164 {165 "pattern": "LinkedInBot",166 "instances": [167 "LinkedInBot/1.0 (compatible; Mozilla/5.0; Jakarta Commons-HttpClient/3.1 +http://www.linkedin.com)",168 "LinkedInBot/1.0 (compatible; Mozilla/5.0; Jakarta Commons-HttpClient/4.3 +http://www.linkedin.com)",169 "LinkedInBot/1.0 (compatible; Mozilla/5.0; Apache-HttpClient +http://www.linkedin.com)"170 ]171 }172 ,173 {174 "pattern": "Python-urllib",175 "instances": [176 "Python-urllib/1.17",177 "Python-urllib/2.5",178 "Python-urllib/2.6",179 "Python-urllib/2.7",180 "Python-urllib/3.1",181 "Python-urllib/3.2",182 "Python-urllib/3.3",183 "Python-urllib/3.4",184 "Python-urllib/3.5",185 "Python-urllib/3.6",186 "Python-urllib/3.7"187 ]188 }189 ,190 {191 "pattern": "python-requests",192 "addition_date": "2018/05/27",193 "instances": [194 "python-requests/2.9.2",195 "python-requests/2.11.1",196 "python-requests/2.18.4",197 "python-requests/2.19.1",198 "python-requests/2.20.0",199 "python-requests/2.21.0",200 "python-requests/2.22.0"201 ]202 }203 ,204 {205 "pattern": "libwww-perl",206 "instances": [207 "2Bone_LinkChecker/1.0 libwww-perl/6.03",208 "2Bone_LinkChkr/1.0 libwww-perl/6.03",209 "amibot - http://www.amidalla.de - tech@amidalla.com libwww-perl/5.831"210 ]211 }212 ,213 {214 "pattern": "httpunit",215 "instances": [216 "httpunit/1.x"217 ]218 }219 ,220 {221 "pattern": "nutch",222 "instances": [223 "NutchCVS/0.7.1 (Nutch; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)",224 "istellabot-nutch/Nutch-1.10"225 ]226 }227 ,228 {229 "pattern": "Go-http-client",230 "addition_date": "2016/03/26",231 "url": "https://golang.org/pkg/net/http/",232 "instances": [233 "Go-http-client/1.1",234 "Go-http-client/2.0"235 ]236 }237 ,238 {239 "pattern": "phpcrawl",240 "addition_date": "2012-09/17",241 "url": "http://phpcrawl.cuab.de/",242 "instances": [243 "phpcrawl"244 ]245 }246 ,247 {248 "pattern": "msnbot",249 "url": "http://search.msn.com/msnbot.htm",250 "instances": [251 "adidxbot/1.1 (+http://search.msn.com/msnbot.htm)",252 "adidxbot/2.0 (+http://search.msn.com/msnbot.htm)",253 "librabot/1.0 (+http://search.msn.com/msnbot.htm)",254 "librabot/2.0 (+http://search.msn.com/msnbot.htm)",255 "msnbot-NewsBlogs/2.0b (+http://search.msn.com/msnbot.htm)",256 "msnbot-UDiscovery/2.0b (+http://search.msn.com/msnbot.htm)",257 "msnbot-media/1.0 (+http://search.msn.com/msnbot.htm)",258 "msnbot-media/1.1 (+http://search.msn.com/msnbot.htm)",259 "msnbot-media/2.0b (+http://search.msn.com/msnbot.htm)",260 "msnbot/1.0 (+http://search.msn.com/msnbot.htm)",261 "msnbot/1.1 (+http://search.msn.com/msnbot.htm)",262 "msnbot/2.0b (+http://search.msn.com/msnbot.htm)",263 "msnbot/2.0b (+http://search.msn.com/msnbot.htm).",264 "msnbot/2.0b (+http://search.msn.com/msnbot.htm)._"265 ]266 }267 ,268 {269 "pattern": "jyxobot",270 "instances": []271 }272 ,273 {274 "pattern": "FAST-WebCrawler",275 "instances": [276 "FAST-WebCrawler/3.6/FirstPage (atw-crawler at fast dot no;http://fast.no/support/crawler.asp)",277 "FAST-WebCrawler/3.7 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)",278 "FAST-WebCrawler/3.7/FirstPage (atw-crawler at fast dot no;http://fast.no/support/crawler.asp)",279 "FAST-WebCrawler/3.8"280 ]281 }282 ,283 {284 "pattern": "FAST Enterprise Crawler",285 "instances": [286 "FAST Enterprise Crawler 6 / Scirus scirus-crawler@fast.no; http://www.scirus.com/srsapp/contactus/",287 "FAST Enterprise Crawler 6 used by Schibsted (webcrawl@schibstedsok.no)"288 ]289 }290 ,291 {292 "pattern": "BIGLOTRON",293 "instances": [294 "BIGLOTRON (Beta 2;GNU/Linux)"295 ]296 }297 ,298 {299 "pattern": "Teoma",300 "instances": [301 "Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://sp.ask.com/docs/about/tech_crawling.html)",302 "Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://about.ask.com/en/docs/about/webmasters.shtml)"303 ],304 "url": "http://about.ask.com/en/docs/about/webmasters.shtml"305 }306 ,307 {308 "pattern": "convera",309 "instances": [310 "ConveraCrawler/0.9e (+http://ews.converasearch.com/crawl.htm)"311 ],312 "url": "http://ews.converasearch.com/crawl.htm"313 }314 ,315 {316 "pattern": "seekbot",317 "instances": [318 "Seekbot/1.0 (http://www.seekbot.net/bot.html) RobotsTxtFetcher/1.2"319 ],320 "url": "http://www.seekbot.net/bot.html"321 }322 ,323 {324 "pattern": "Gigabot",325 "instances": [326 "Gigabot/1.0",327 "Gigabot/2.0 (http://www.gigablast.com/spider.html)"328 ],329 "url": "http://www.gigablast.com/spider.html"330 }331 ,332 {333 "pattern": "Gigablast",334 "instances": [335 "GigablastOpenSource/1.0"336 ],337 "url": "https://github.com/gigablast/open-source-search-engine"338 }339 ,340 {341 "pattern": "exabot",342 "instances": [343 "Mozilla/5.0 (compatible; Alexabot/1.0; +http://www.alexa.com/help/certifyscan; certifyscan@alexa.com)",344 "Mozilla/5.0 (compatible; Exabot PyExalead/3.0; +http://www.exabot.com/go/robot)",345 "Mozilla/5.0 (compatible; Exabot-Images/3.0; +http://www.exabot.com/go/robot)",346 "Mozilla/5.0 (compatible; Exabot/3.0 (BiggerBetter); +http://www.exabot.com/go/robot)",347 "Mozilla/5.0 (compatible; Exabot/3.0; +http://www.exabot.com/go/robot)",348 "Mozilla/5.0 (compatible; Exabot/3.0; http://www.exabot.com/go/robot)"349 ]350 }351 ,352 {353 "pattern": "ia_archiver",354 "instances": [355 "ia_archiver (+http://www.alexa.com/site/help/webmasters; crawler@alexa.com)",356 "ia_archiver-web.archive.org"357 ]358 }359 ,360 {361 "pattern": "GingerCrawler",362 "instances": [363 "GingerCrawler/1.0 (Language Assistant for Dyslexics; www.gingersoftware.com/crawler_agent.htm; support at ginger software dot com)"364 ]365 }366 ,367 {368 "pattern": "webmon ",369 "instances": []370 }371 ,372 {373 "pattern": "HTTrack",374 "instances": [375 "Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98)"376 ]377 }378 ,379 {380 "pattern": "grub.org",381 "instances": [382 "Mozilla/4.0 (compatible; grub-client-0.3.0; Crawl your own stuff with http://grub.org)",383 "Mozilla/4.0 (compatible; grub-client-1.0.4; Crawl your own stuff with http://grub.org)",384 "Mozilla/4.0 (compatible; grub-client-1.0.5; Crawl your own stuff with http://grub.org)",385 "Mozilla/4.0 (compatible; grub-client-1.0.6; Crawl your own stuff with http://grub.org)",386 "Mozilla/4.0 (compatible; grub-client-1.0.7; Crawl your own stuff with http://grub.org)",387 "Mozilla/4.0 (compatible; grub-client-1.1.1; Crawl your own stuff with http://grub.org)",388 "Mozilla/4.0 (compatible; grub-client-1.2.1; Crawl your own stuff with http://grub.org)",389 "Mozilla/4.0 (compatible; grub-client-1.3.1; Crawl your own stuff with http://grub.org)",390 "Mozilla/4.0 (compatible; grub-client-1.3.7; Crawl your own stuff with http://grub.org)",391 "Mozilla/4.0 (compatible; grub-client-1.4.3; Crawl your own stuff with http://grub.org)",392 "Mozilla/4.0 (compatible; grub-client-1.5.3; Crawl your own stuff with http://grub.org)"393 ]394 }395 ,396 {397 "pattern": "UsineNouvelleCrawler",398 "instances": []399 }400 ,401 {402 "pattern": "antibot",403 "instances": []404 }405 ,406 {407 "pattern": "netresearchserver",408 "instances": []409 }410 ,411 {412 "pattern": "speedy",413 "instances": [414 "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) Speedy Spider (http://www.entireweb.com/about/search_tech/speedy_spider/)",415 "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) Speedy Spider for SpeedyAds (http://www.entireweb.com/about/search_tech/speedy_spider/)",416 "Mozilla/5.0 (compatible; Speedy Spider; http://www.entireweb.com/about/search_tech/speedy_spider/)",417 "Speedy Spider (Entireweb; Beta/1.2; http://www.entireweb.com/about/search_tech/speedyspider/)",418 "Speedy Spider (http://www.entireweb.com/about/search_tech/speedy_spider/)"419 ]420 }421 ,422 {423 "pattern": "fluffy",424 "instances": []425 }426 ,427 {428 "pattern": "findlink",429 "instances": [430 "findlinks/1.0 (+http://wortschatz.uni-leipzig.de/findlinks/)",431 "findlinks/1.1.3-beta8 (+http://wortschatz.uni-leipzig.de/findlinks/)",432 "findlinks/1.1.3-beta9 (+http://wortschatz.uni-leipzig.de/findlinks/)",433 "findlinks/1.1.5-beta7 (+http://wortschatz.uni-leipzig.de/findlinks/)",434 "findlinks/1.1.6-beta1 (+http://wortschatz.uni-leipzig.de/findlinks/)",435 "findlinks/1.1.6-beta1 (+http://wortschatz.uni-leipzig.de/findlinks/; YaCy 0.1; yacy.net)",436 "findlinks/1.1.6-beta2 (+http://wortschatz.uni-leipzig.de/findlinks/)",437 "findlinks/1.1.6-beta3 (+http://wortschatz.uni-leipzig.de/findlinks/)",438 "findlinks/1.1.6-beta4 (+http://wortschatz.uni-leipzig.de/findlinks/)",439 "findlinks/1.1.6-beta5 (+http://wortschatz.uni-leipzig.de/findlinks/)",440 "findlinks/1.1.6-beta6 (+http://wortschatz.uni-leipzig.de/findlinks/)",441 "findlinks/2.0 (+http://wortschatz.uni-leipzig.de/findlinks/)",442 "findlinks/2.0.1 (+http://wortschatz.uni-leipzig.de/findlinks/)",443 "findlinks/2.0.2 (+http://wortschatz.uni-leipzig.de/findlinks/)",444 "findlinks/2.0.4 (+http://wortschatz.uni-leipzig.de/findlinks/)",445 "findlinks/2.0.5 (+http://wortschatz.uni-leipzig.de/findlinks/)",446 "findlinks/2.0.9 (+http://wortschatz.uni-leipzig.de/findlinks/)",447 "findlinks/2.1 (+http://wortschatz.uni-leipzig.de/findlinks/)",448 "findlinks/2.1.3 (+http://wortschatz.uni-leipzig.de/findlinks/)",449 "findlinks/2.1.5 (+http://wortschatz.uni-leipzig.de/findlinks/)",450 "findlinks/2.2 (+http://wortschatz.uni-leipzig.de/findlinks/)",451 "findlinks/2.5 (+http://wortschatz.uni-leipzig.de/findlinks/)",452 "findlinks/2.6 (+http://wortschatz.uni-leipzig.de/findlinks/)"453 ]454 }455 ,456 {457 "pattern": "msrbot",458 "instances": []459 }460 ,461 {462 "pattern": "panscient",463 "instances": [464 "panscient.com"465 ]466 }467 ,468 {469 "pattern": "yacybot",470 "instances": [471 "yacybot (/global; amd64 FreeBSD 10.3-RELEASE; java 1.8.0_77; GMT/en) http://yacy.net/bot.html",472 "yacybot (/global; amd64 FreeBSD 10.3-RELEASE-p7; java 1.7.0_95; GMT/en) http://yacy.net/bot.html",473 "yacybot (-global; amd64 FreeBSD 9.2-RELEASE-p10; java 1.7.0_65; Europe/en) http://yacy.net/bot.html",474 "yacybot (/global; amd64 Linux 2.6.32-042stab093.4; java 1.7.0_65; Etc/en) http://yacy.net/bot.html",475 "yacybot (/global; amd64 Linux 2.6.32-042stab094.8; java 1.7.0_79; America/en) http://yacy.net/bot.html",476 "yacybot (/global; amd64 Linux 2.6.32-042stab108.8; java 1.7.0_91; America/en) http://yacy.net/bot.html",477 "yacybot (-global; amd64 Linux 2.6.32-042stab111.11; java 1.7.0_79; Europe/en) http://yacy.net/bot.html",478 "yacybot (-global; amd64 Linux 2.6.32-042stab116.1; java 1.7.0_79; Europe/en) http://yacy.net/bot.html",479 "yacybot (/global; amd64 Linux 2.6.32-573.3.1.el6.x86_64; java 1.7.0_85; Europe/en) http://yacy.net/bot.html",480 "yacybot (-global; amd64 Linux 3.10.0-229.4.2.el7.x86_64; java 1.7.0_79; Europe/en) http://yacy.net/bot.html",481 "yacybot (-global; amd64 Linux 3.10.0-229.4.2.el7.x86_64; java 1.8.0_45; Europe/en) http://yacy.net/bot.html",482 "yacybot (/global; amd64 Linux 3.10.0-229.7.2.el7.x86_64; java 1.8.0_45; Europe/en) http://yacy.net/bot.html",483 "yacybot (/global; amd64 Linux 3.10.0-327.22.2.el7.x86_64; java 1.7.0_101; Etc/en) http://yacy.net/bot.html",484 "yacybot (/global; amd64 Linux 3.11.10-21-desktop; java 1.7.0_51; America/en) http://yacy.net/bot.html",485 "yacybot (/global; amd64 Linux 3.12.1; java 1.7.0_65; Europe/en) http://yacy.net/bot.html",486 "yacybot (/global; amd64 Linux 3.13.0-042stab093.4; java 1.7.0_79; Europe/de) http://yacy.net/bot.html",487 "yacybot (/global; amd64 Linux 3.13.0-042stab093.4; java 1.7.0_79; Europe/en) http://yacy.net/bot.html",488 "yacybot (/global; amd64 Linux 3.13.0-45-generic; java 1.7.0_75; Europe/en) http://yacy.net/bot.html",489 "yacybot (-global; amd64 Linux 3.13.0-61-generic; java 1.7.0_79; Europe/en) http://yacy.net/bot.html",490 "yacybot (/global; amd64 Linux 3.13.0-74-generic; java 1.7.0_91; Europe/en) http://yacy.net/bot.html",491 "yacybot (/global; amd64 Linux 3.13.0-83-generic; java 1.7.0_95; Europe/de) http://yacy.net/bot.html",492 "yacybot (/global; amd64 Linux 3.13.0-83-generic; java 1.7.0_95; Europe/en) http://yacy.net/bot.html",493 "yacybot (/global; amd64 Linux 3.13.0-85-generic; java 1.7.0_101; Europe/en) http://yacy.net/bot.html",494 "yacybot (/global; amd64 Linux 3.13.0-85-generic; java 1.7.0_95; Europe/en) http://yacy.net/bot.html",495 "yacybot (/global; amd64 Linux 3.13.0-88-generic; java 1.7.0_101; Europe/en) http://yacy.net/bot.html",496 "yacybot (/global; amd64 Linux 3.14-0.bpo.1-amd64; java 1.7.0_55; Europe/de) http://yacy.net/bot.html",497 "yacybot (/global; amd64 Linux 3.14.32-xxxx-grs-ipv6-64; java 1.7.0_75; Europe/en) http://yacy.net/bot.html",498 "yacybot (-global; amd64 Linux 3.14.32-xxxx-grs-ipv6-64; java 1.8.0_111; Europe/de) http://yacy.net/bot.html",499 "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_111; Europe/de) http://yacy.net/bot.html",500 "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_75; America/en) http://yacy.net/bot.html",501 "yacybot (-global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_75; Europe/en) http://yacy.net/bot.html",502 "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_75; Europe/en) http://yacy.net/bot.html",503 "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_79; Europe/de) http://yacy.net/bot.html",504 "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_79; Europe/en) http://yacy.net/bot.html",505 "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_91; Europe/de) http://yacy.net/bot.html",506 "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_95; Europe/en) http://yacy.net/bot.html",507 "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.8.0_111; Europe/en) http://yacy.net/bot.html",508 "yacybot (/global; amd64 Linux 3.16-0.bpo.2-amd64; java 1.7.0_65; Europe/en) http://yacy.net/bot.html",509 "yacybot (-global; amd64 Linux 3.19.0-15-generic; java 1.8.0_45-internal; Europe/de) http://yacy.net/bot.html",510 "yacybot (-global; amd64 Linux 3.2.0-4-amd64; java 1.7.0_65; Europe/en) http://yacy.net/bot.html",511 "yacybot (-global; amd64 Linux 3.2.0-4-amd64; java 1.7.0_67; Europe/en) http://yacy.net/bot.html",512 "yacybot (-global; amd64 Linux 4.4.0-57-generic; java 9-internal; Europe/en) http://yacy.net/bot.html",513 "yacybot (-global; amd64 Windows 8.1 6.3; java 1.7.0_55; Europe/de) http://yacy.net/bot.html",514 "yacybot (-global; amd64 Windows 8 6.2; java 1.7.0_55; Europe/de) http://yacy.net/bot.html",515 "yacybot (-global; amd64 Linux 5.2.8-Jinsol; java 12.0.2; Europe/en) http://yacy.net/bot.html",516 "yacybot (-global; amd64 Linux 5.2.9-Jinsol; java 12.0.2; Europe/en) http://yacy.net/bot.html",517 "yacybot (-global; amd64 Linux 5.2.11-Jinsol; java 12.0.2; Europe/en) http://yacy.net/bot.html"518 ]519 }520 ,521 {522 "pattern": "AISearchBot",523 "instances": []524 }525 ,526 {527 "pattern": "ips-agent",528 "instances": [529 "BlackBerry9000/4.6.0.167 Profile/MIDP-2.0 Configuration/CLDC-1.1 VendorID/102 ips-agent",530 "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.12; ips-agent) Gecko/20050922 Fedora/1.0.7-1.1.fc4 Firefox/1.0.7",531 "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.1.3; ips-agent) Gecko/20090824 Fedora/1.0.7-1.1.fc4 Firefox/3.5.3",532 "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.24; ips-agent) Gecko/20111107 Ubuntu/10.04 (lucid) Firefox/3.6.24",533 "Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:14.0; ips-agent) Gecko/20100101 Firefox/14.0.1"534 ]535 }536 ,537 {538 "pattern": "tagoobot",539 "instances": []540 }541 ,542 {543 "pattern": "MJ12bot",544 "instances": [545 "MJ12bot/v1.2.0 (http://majestic12.co.uk/bot.php?+)",546 "Mozilla/5.0 (compatible; MJ12bot/v1.2.1; http://www.majestic12.co.uk/bot.php?+)",547 "Mozilla/5.0 (compatible; MJ12bot/v1.2.3; http://www.majestic12.co.uk/bot.php?+)",548 "Mozilla/5.0 (compatible; MJ12bot/v1.2.4; http://www.majestic12.co.uk/bot.php?+)",549 "Mozilla/5.0 (compatible; MJ12bot/v1.2.5; http://www.majestic12.co.uk/bot.php?+)",550 "Mozilla/5.0 (compatible; MJ12bot/v1.3.0; http://www.majestic12.co.uk/bot.php?+)",551 "Mozilla/5.0 (compatible; MJ12bot/v1.3.1; http://www.majestic12.co.uk/bot.php?+)",552 "Mozilla/5.0 (compatible; MJ12bot/v1.3.2; http://www.majestic12.co.uk/bot.php?+)",553 "Mozilla/5.0 (compatible; MJ12bot/v1.3.3; http://www.majestic12.co.uk/bot.php?+)",554 "Mozilla/5.0 (compatible; MJ12bot/v1.4.0; http://www.majestic12.co.uk/bot.php?+)",555 "Mozilla/5.0 (compatible; MJ12bot/v1.4.1; http://www.majestic12.co.uk/bot.php?+)",556 "Mozilla/5.0 (compatible; MJ12bot/v1.4.2; http://www.majestic12.co.uk/bot.php?+)",557 "Mozilla/5.0 (compatible; MJ12bot/v1.4.3; http://www.majestic12.co.uk/bot.php?+)",558 "Mozilla/5.0 (compatible; MJ12bot/v1.4.4 (domain ownership verifier); http://www.majestic12.co.uk/bot.php?+)",559 "Mozilla/5.0 (compatible; MJ12bot/v1.4.4; http://www.majestic12.co.uk/bot.php?+)",560 "Mozilla/5.0 (compatible; MJ12bot/v1.4.5; http://www.majestic12.co.uk/bot.php?+)",561 "Mozilla/5.0 (compatible; MJ12bot/v1.4.6; http://mj12bot.com/)",562 "Mozilla/5.0 (compatible; MJ12bot/v1.4.7; http://mj12bot.com/)",563 "Mozilla/5.0 (compatible; MJ12bot/v1.4.7; http://www.majestic12.co.uk/bot.php?+)",564 "Mozilla/5.0 (compatible; MJ12bot/v1.4.8; http://mj12bot.com/)"565 ]566 }567 ,568 {569 "pattern": "woriobot",570 "instances": [571 "Mozilla/5.0 (compatible; woriobot +http://worio.com)",572 "Mozilla/5.0 (compatible; woriobot support [at] zite [dot] com +http://zite.com)"573 ]574 }575 ,576 {577 "pattern": "yanga",578 "instances": [579 "Yanga WorldSearch Bot v1.1/beta (http://www.yanga.co.uk/)"580 ]581 }582 ,583 {584 "pattern": "buzzbot",585 "instances": [586 "Buzzbot/1.0 (Buzzbot; http://www.buzzstream.com; buzzbot@buzzstream.com)"587 ]588 }589 ,590 {591 "pattern": "mlbot",592 "instances": [593 "MLBot (www.metadatalabs.com/mlbot)"594 ]595 }596 ,597 {598 "pattern": "YandexBot",599 "url": "http://yandex.com/bots",600 "instances": [601 "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)"602 ],603 "addition_date": "2015/04/14"604 }605 ,606 {607 "pattern": "YandexImages",608 "url": "http://yandex.com/bots",609 "instances": [610 "Mozilla/5.0 (compatible; YandexImages/3.0; +http://yandex.com/bots)"611 ],612 "addition_date": "2015/04/14"613 }614 ,615 {616 "pattern": "YandexAccessibilityBot",617 "url": "http://yandex.com/bots",618 "instances": [619 "Mozilla/5.0 (compatible; YandexAccessibilityBot/3.0; +http://yandex.com/bots"620 ],621 "addition_date": "2019/03/01"622 }623 ,624 {625 "pattern": "YandexMobileBot",626 "url": "https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.xml#robot-in-logs",627 "instances": [628 "Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B411 Safari/600.1.4 (compatible; YandexMobileBot/3.0; +http://yandex.com/bots)"629 ],630 "addition_date": "2016/12/01"631 }632 ,633 {634 "pattern": "purebot",635 "addition_date": "2010/01/19",636 "instances": []637 }638 ,639 {640 "pattern": "Linguee Bot",641 "addition_date": "2010/01/26",642 "url": "http://www.linguee.com/bot",643 "instances": [644 "Linguee Bot (http://www.linguee.com/bot)",645 "Linguee Bot (http://www.linguee.com/bot; bot@linguee.com)"646 ]647 }648 ,649 {650 "pattern": "CyberPatrol",651 "addition_date": "2010/02/11",652 "url": "http://www.cyberpatrol.com/cyberpatrolcrawler.asp",653 "instances": [654 "CyberPatrol SiteCat Webbot (http://www.cyberpatrol.com/cyberpatrolcrawler.asp)"655 ]656 }657 ,658 {659 "pattern": "voilabot",660 "addition_date": "2010/05/18",661 "instances": [662 "Mozilla/5.0 (Windows NT 5.1; U; Win64; fr; rv:1.8.1) VoilaBot BETA 1.2 (support.voilabot@orange-ftgroup.com)",663 "Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.8.1) VoilaBot BETA 1.2 (support.voilabot@orange-ftgroup.com)"664 ]665 }666 ,667 {668 "pattern": "Baiduspider",669 "addition_date": "2010/07/15",670 "url": "http://www.baidu.jp/spider/",671 "instances": [672 "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)",673 "Mozilla/5.0 (compatible; Baiduspider-render/2.0; +http://www.baidu.com/search/spider.html)"674 ]675 }676 ,677 {678 "pattern": "citeseerxbot",679 "addition_date": "2010/07/17",680 "instances": []681 }682 ,683 {684 "pattern": "spbot",685 "addition_date": "2010/07/31",686 "url": "http://www.seoprofiler.com/bot",687 "instances": [688 "Mozilla/5.0 (compatible; spbot/1.0; +http://www.seoprofiler.com/bot/ )",689 "Mozilla/5.0 (compatible; spbot/1.1; +http://www.seoprofiler.com/bot/ )",690 "Mozilla/5.0 (compatible; spbot/1.2; +http://www.seoprofiler.com/bot/ )",691 "Mozilla/5.0 (compatible; spbot/2.0.1; +http://www.seoprofiler.com/bot/ )",692 "Mozilla/5.0 (compatible; spbot/2.0.2; +http://www.seoprofiler.com/bot/ )",693 "Mozilla/5.0 (compatible; spbot/2.0.3; +http://www.seoprofiler.com/bot/ )",694 "Mozilla/5.0 (compatible; spbot/2.0.4; +http://www.seoprofiler.com/bot )",695 "Mozilla/5.0 (compatible; spbot/2.0; +http://www.seoprofiler.com/bot/ )",696 "Mozilla/5.0 (compatible; spbot/2.1; +http://www.seoprofiler.com/bot )",697 "Mozilla/5.0 (compatible; spbot/3.0; +http://www.seoprofiler.com/bot )",698 "Mozilla/5.0 (compatible; spbot/3.1; +http://www.seoprofiler.com/bot )",699 "Mozilla/5.0 (compatible; spbot/4.0.1; +http://www.seoprofiler.com/bot )",700 "Mozilla/5.0 (compatible; spbot/4.0.2; +http://www.seoprofiler.com/bot )",701 "Mozilla/5.0 (compatible; spbot/4.0.3; +http://www.seoprofiler.com/bot )",702 "Mozilla/5.0 (compatible; spbot/4.0.4; +http://www.seoprofiler.com/bot )",703 "Mozilla/5.0 (compatible; spbot/4.0.5; +http://www.seoprofiler.com/bot )",704 "Mozilla/5.0 (compatible; spbot/4.0.6; +http://www.seoprofiler.com/bot )",705 "Mozilla/5.0 (compatible; spbot/4.0.7; +http://OpenLinkProfiler.org/bot )",706 "Mozilla/5.0 (compatible; spbot/4.0.7; +https://www.seoprofiler.com/bot )",707 "Mozilla/5.0 (compatible; spbot/4.0.8; +http://OpenLinkProfiler.org/bot )",708 "Mozilla/5.0 (compatible; spbot/4.0.9; +http://OpenLinkProfiler.org/bot )",709 "Mozilla/5.0 (compatible; spbot/4.0; +http://www.seoprofiler.com/bot )",710 "Mozilla/5.0 (compatible; spbot/4.0a; +http://www.seoprofiler.com/bot )",711 "Mozilla/5.0 (compatible; spbot/4.0b; +http://www.seoprofiler.com/bot )",712 "Mozilla/5.0 (compatible; spbot/4.1.0; +http://OpenLinkProfiler.org/bot )",713 "Mozilla/5.0 (compatible; spbot/4.2.0; +http://OpenLinkProfiler.org/bot )",714 "Mozilla/5.0 (compatible; spbot/4.3.0; +http://OpenLinkProfiler.org/bot )",715 "Mozilla/5.0 (compatible; spbot/4.4.0; +http://OpenLinkProfiler.org/bot )",716 "Mozilla/5.0 (compatible; spbot/4.4.1; +http://OpenLinkProfiler.org/bot )",717 "Mozilla/5.0 (compatible; spbot/4.4.2; +http://OpenLinkProfiler.org/bot )",718 "Mozilla/5.0 (compatible; spbot/5.0.1; +http://OpenLinkProfiler.org/bot )",719 "Mozilla/5.0 (compatible; spbot/5.0.2; +http://OpenLinkProfiler.org/bot )",720 "Mozilla/5.0 (compatible; spbot/5.0.3; +http://OpenLinkProfiler.org/bot )",721 "Mozilla/5.0 (compatible; spbot/5.0; +http://OpenLinkProfiler.org/bot )"722 ]723 }724 ,725 {726 "pattern": "twengabot",727 "addition_date": "2010/08/03",728 "url": "http://www.twenga.com/bot.html",729 "instances": []730 }731 ,732 {733 "pattern": "postrank",734 "addition_date": "2010/08/03",735 "url": "http://www.postrank.com",736 "instances": [737 "PostRank/2.0 (postrank.com)",738 "PostRank/2.0 (postrank.com; 1 subscribers)"739 ]740 }741 ,742 {743 "pattern": "TurnitinBot",744 "addition_date": "2010/09/26",745 "url": "http://www.turnitin.com",746 "instances": [747 "TurnitinBot (https://turnitin.com/robot/crawlerinfo.html)"748 ]749 }750 ,751 {752 "pattern": "scribdbot",753 "addition_date": "2010/09/28",754 "url": "http://www.scribd.com",755 "instances": []756 }757 ,758 {759 "pattern": "page2rss",760 "addition_date": "2010/10/07",761 "url": "http://www.page2rss.com",762 "instances": [763 "Mozilla/5.0 (compatible; Page2RSS/0.7; +http://page2rss.com/)"764 ]765 }766 ,767 {768 "pattern": "sitebot",769 "addition_date": "2010/12/15",770 "url": "http://www.sitebot.org",771 "instances": [772 "Mozilla/5.0 (compatible; Whoiswebsitebot/0.1; +http://www.whoiswebsite.net)"773 ]774 }775 ,776 {777 "pattern": "linkdex",778 "addition_date": "2011/01/06",779 "url": "http://www.linkdex.com",780 "instances": [781 "Mozilla/5.0 (compatible; linkdexbot/2.0; +http://www.linkdex.com/about/bots/)",782 "Mozilla/5.0 (compatible; linkdexbot/2.0; +http://www.linkdex.com/bots/)",783 "Mozilla/5.0 (compatible; linkdexbot/2.1; +http://www.linkdex.com/about/bots/)",784 "Mozilla/5.0 (compatible; linkdexbot/2.1; +http://www.linkdex.com/bots/)",785 "Mozilla/5.0 (compatible; linkdexbot/2.2; +http://www.linkdex.com/bots/)",786 "linkdex.com/v2.0",787 "linkdexbot/Nutch-1.0-dev (http://www.linkdex.com/; crawl at linkdex dot com)"788 ]789 }790 ,791 {792 "pattern": "Adidxbot",793 "url": "http://onlinehelp.microsoft.com/en-us/bing/hh204496.aspx",794 "instances": []795 }796 ,797 {798 "pattern": "ezooms",799 "addition_date": "2011/04/27",800 "url": "http://www.phpbb.com/community/viewtopic.php?f=64&t=935605&start=450#p12948289",801 "instances": [802 "Mozilla/5.0 (compatible; Ezooms/1.0; ezooms.bot@gmail.com)"803 ]804 }805 ,806 {807 "pattern": "dotbot",808 "addition_date": "2011/04/27",809 "instances": [810 "Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, help@moz.com)",811 "dotbot"812 ]813 }814 ,815 {816 "pattern": "Mail.RU_Bot",817 "addition_date": "2011/04/27",818 "instances": [819 "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/help/robots)",820 "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/",821 "Mozilla/5.0 (compatible; Mail.RU_Bot/2.0; +http://go.mail.ru/",822 "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/Robots/2.0; +http://go.mail.ru/help/robots)"823 ]824 }825 ,826 {827 "pattern": "discobot",828 "addition_date": "2011/05/03",829 "url": "http://discoveryengine.com/discobot.html",830 "instances": [831 "Mozilla/5.0 (compatible; discobot/1.0; +http://discoveryengine.com/discobot.html)",832 "Mozilla/5.0 (compatible; discobot/2.0; +http://discoveryengine.com/discobot.html)",833 "mozilla/5.0 (compatible; discobot/1.1; +http://discoveryengine.com/discobot.html)"834 ]835 }836 ,837 {838 "pattern": "heritrix",839 "addition_date": "2011/06/21",840 "url": "https://github.com/internetarchive/heritrix3/wiki",841 "instances": [842 "Mozilla/5.0 (compatible; heritrix/1.12.1 +http://www.webarchiv.cz)",843 "Mozilla/5.0 (compatible; heritrix/1.12.1b +http://netarkivet.dk/website/info.html)",844 "Mozilla/5.0 (compatible; heritrix/1.14.2 +http://rjpower.org)",845 "Mozilla/5.0 (compatible; heritrix/1.14.2 +http://www.webarchiv.cz)",846 "Mozilla/5.0 (compatible; heritrix/1.14.3 +http://archive.org)",847 "Mozilla/5.0 (compatible; heritrix/1.14.3 +http://www.accelobot.com)",848 "Mozilla/5.0 (compatible; heritrix/1.14.3 +http://www.webarchiv.cz)",849 "Mozilla/5.0 (compatible; heritrix/1.14.3.r6601 +http://www.buddybuzz.net/yptrino)",850 "Mozilla/5.0 (compatible; heritrix/1.14.4 +http://parsijoo.ir)",851 "Mozilla/5.0 (compatible; heritrix/1.14.4 +http://www.exif-search.com)",852 "Mozilla/5.0 (compatible; heritrix/2.0.2 +http://aihit.com)",853 "Mozilla/5.0 (compatible; heritrix/2.0.2 +http://seekda.com)",854 "Mozilla/5.0 (compatible; heritrix/3.0.0-SNAPSHOT-20091120.021634 +http://crawler.archive.org)",855 "Mozilla/5.0 (compatible; heritrix/3.1.0-RC1 +http://boston.lti.cs.cmu.edu/crawler_12/)",856 "Mozilla/5.0 (compatible; heritrix/3.1.1 +http://places.tomtom.com/crawlerinfo)",857 "Mozilla/5.0 (compatible; heritrix/3.1.1 +http://www.mixdata.com)",858 "Mozilla/5.0 (compatible; heritrix/3.1.1; UniLeipzigASV +http://corpora.informatik.uni-leipzig.de/crawler_faq.html)",859 "Mozilla/5.0 (compatible; heritrix/3.2.0 +http://www.crim.ca)",860 "Mozilla/5.0 (compatible; heritrix/3.2.0 +http://www.exif-search.com)",861 "Mozilla/5.0 (compatible; heritrix/3.2.0 +http://www.mixdata.com)",862 "Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-20160309-0050; UniLeipzigASV +http://corpora.informatik.uni-leipzig.de/crawler_faq.html)",863 "Mozilla/5.0 (compatible; sukibot_heritrix/3.1.1 +http://suki.ling.helsinki.fi/eng/webmasters.html)"864 ]865 }866 ,867 {868 "pattern": "findthatfile",869 "addition_date": "2011/06/21",870 "url": "http://www.findthatfile.com/",871 "instances": []872 }873 ,874 {875 "pattern": "europarchive.org",876 "addition_date": "2011/06/21",877 "url": "",878 "instances": [879 "Mozilla/5.0 (compatible; MSIE 7.0 +http://www.europarchive.org)"880 ]881 }882 ,883 {884 "pattern": "NerdByNature.Bot",885 "addition_date": "2011/07/12",886 "url": "http://www.nerdbynature.net/bot",887 "instances": [888 "Mozilla/5.0 (compatible; NerdByNature.Bot; http://www.nerdbynature.net/bot)"889 ]890 }891 ,892 {893 "pattern": "sistrix crawler",894 "addition_date": "2011/08/02",895 "instances": []896 }897 ,898 {899 "pattern": "Ahrefs(Bot|SiteAudit)",900 "addition_date": "2011/08/28",901 "instances": [902 "Mozilla/5.0 (compatible; AhrefsBot/6.1; +http://ahrefs.com/robot/)",903 "Mozilla/5.0 (compatible; AhrefsSiteAudit/6.1; +http://ahrefs.com/robot/)",904 "Mozilla/5.0 (compatible; AhrefsBot/5.2; News; +http://ahrefs.com/robot/)",905 "Mozilla/5.0 (compatible; AhrefsBot/5.2; +http://ahrefs.com/robot/)",906 "Mozilla/5.0 (compatible; AhrefsSiteAudit/5.2; +http://ahrefs.com/robot/)",907 "Mozilla/5.0 (compatible; AhrefsBot/6.1; News; +http://ahrefs.com/robot/)"908 ]909 }910 ,911 {912 "pattern": "fuelbot",913 "addition_date": "2018/06/28",914 "instances": [915 "fuelbot"916 ]917 }918 ,919 {920 "pattern": "CrunchBot",921 "addition_date": "2018/06/28",922 "instances": [923 "CrunchBot/1.0 (+http://www.leadcrunch.com/crunchbot)"924 ]925 }926 ,927 {928 "pattern": "IndeedBot",929 "addition_date": "2018/06/28",930 "instances": [931 "Mozilla/5.0 (Windows NT 6.1; rv:38.0) Gecko/20100101 Firefox/38.0 (IndeedBot 1.1)"932 ]933 }934 ,935 {936 "pattern": "mappydata",937 "addition_date": "2018/06/28",938 "instances": [939 "Mozilla/5.0 (compatible; Mappy/1.0; +http://mappydata.net/bot/)"940 ]941 }942 ,943 {944 "pattern": "woobot",945 "addition_date": "2018/06/28",946 "instances": [947 "woobot"948 ]949 }950 ,951 {952 "pattern": "ZoominfoBot",953 "addition_date": "2018/06/28",954 "instances": [955 "ZoominfoBot (zoominfobot at zoominfo dot com)"956 ]957 }958 ,959 {960 "pattern": "PrivacyAwareBot",961 "addition_date": "2018/06/28",962 "instances": [963 "Mozilla/5.0 (compatible; PrivacyAwareBot/1.1; +http://www.privacyaware.org)"964 ]965 }966 ,967 {968 "pattern": "Multiviewbot",969 "addition_date": "2018/06/28",970 "instances": [971 "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Multiviewbot"972 ]973 }974 ,975 {976 "pattern": "SWIMGBot",977 "addition_date": "2018/06/28",978 "instances": [979 "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36 SWIMGBot"980 ]981 }982 ,983 {984 "pattern": "Grobbot",985 "addition_date": "2018/06/28",986 "instances": [987 "Mozilla/5.0 (compatible; Grobbot/2.2; +https://grob.it)"988 ]989 }990 ,991 {992 "pattern": "eright",993 "addition_date": "2018/06/28",994 "instances": [995 "Mozilla/5.0 (compatible; eright/1.0; +bot@eright.com)"996 ]997 }998 ,999 {1000 "pattern": "Apercite",1001 "addition_date": "2018/06/28",1002 "instances": [1003 "Mozilla/5.0 (compatible; Apercite; +http://www.apercite.fr/robot/index.html)"1004 ]1005 }1006 ,1007 {1008 "pattern": "semanticbot",1009 "addition_date": "2018/06/28",1010 "instances": [1011 "semanticbot",1012 "semanticbot (info@semanticaudience.com)"1013 ]1014 }1015 ,1016 {1017 "pattern": "Aboundex",1018 "addition_date": "2011/09/28",1019 "url": "http://www.aboundex.com/crawler/",1020 "instances": [1021 "Aboundex/0.2 (http://www.aboundex.com/crawler/)",1022 "Aboundex/0.3 (http://www.aboundex.com/crawler/)"1023 ]1024 }1025 ,1026 {1027 "pattern": "domaincrawler",1028 "addition_date": "2011/10/21",1029 "instances": [1030 "CipaCrawler/3.0 (info@domaincrawler.com; http://www.domaincrawler.com/www.example.com)"1031 ]1032 }1033 ,1034 {1035 "pattern": "wbsearchbot",1036 "addition_date": "2011/12/21",1037 "url": "http://www.warebay.com/bot.html",1038 "instances": []1039 }1040 ,1041 {1042 "pattern": "summify",1043 "addition_date": "2012/01/04",1044 "url": "http://summify.com",1045 "instances": [1046 "Summify (Summify/1.0.1; +http://summify.com)"1047 ]1048 }1049 ,1050 {1051 "pattern": "CCBot",1052 "addition_date": "2012/02/05",1053 "url": "http://www.commoncrawl.org/bot.html",1054 "instances": [1055 "CCBot/2.0 (http://commoncrawl.org/faq/)",1056 "CCBot/2.0 (https://commoncrawl.org/faq/)"1057 ]1058 }1059 ,1060 {1061 "pattern": "edisterbot",1062 "addition_date": "2012/02/25",1063 "instances": []1064 }1065 ,1066 {1067 "pattern": "seznambot",1068 "addition_date": "2012/03/14",1069 "instances": [1070 "Mozilla/5.0 (compatible; SeznamBot/3.2-test1-1; +http://napoveda.seznam.cz/en/seznambot-intro/)",1071 "Mozilla/5.0 (compatible; SeznamBot/3.2-test1; +http://napoveda.seznam.cz/en/seznambot-intro/)",1072 "Mozilla/5.0 (compatible; SeznamBot/3.2-test2; +http://napoveda.seznam.cz/en/seznambot-intro/)",1073 "Mozilla/5.0 (compatible; SeznamBot/3.2-test4; +http://napoveda.seznam.cz/en/seznambot-intro/)",1074 "Mozilla/5.0 (compatible; SeznamBot/3.2; +http://napoveda.seznam.cz/en/seznambot-intro/)"1075 ]1076 }1077 ,1078 {1079 "pattern": "ec2linkfinder",1080 "addition_date": "2012/03/22",1081 "instances": [1082 "ec2linkfinder"1083 ]1084 }1085 ,1086 {1087 "pattern": "gslfbot",1088 "addition_date": "2012/04/03",1089 "instances": []1090 }1091 ,1092 {1093 "pattern": "aiHitBot",1094 "addition_date": "2012/04/16",1095 "instances": [1096 "Mozilla/5.0 (compatible; aiHitBot/2.9; +https://www.aihitdata.com/about)"1097 ]1098 }1099 ,1100 {1101 "pattern": "intelium_bot",1102 "addition_date": "2012/05/07",1103 "instances": []1104 }1105 ,1106 {1107 "pattern": "facebookexternalhit",1108 "addition_date": "2012/05/07",1109 "instances": [1110 "facebookexternalhit/1.0 (+http://www.facebook.com/externalhit_uatext.php)",1111 "facebookexternalhit/1.1",1112 "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)"1113 ],1114 "url": "https://developers.facebook.com/docs/sharing/webmasters/crawler/"1115 }1116 ,1117 {1118 "pattern": "Yeti",1119 "addition_date": "2012/05/07",1120 "url": "http://naver.me/bot",1121 "instances": [1122 "Mozilla/5.0 (compatible; Yeti/1.1; +http://naver.me/bot)"1123 ]1124 }1125 ,1126 {1127 "pattern": "RetrevoPageAnalyzer",1128 "addition_date": "2012/05/07",1129 "instances": [1130 "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; RetrevoPageAnalyzer; +http://www.retrevo.com/content/about-us)"1131 ]1132 }1133 ,1134 {1135 "pattern": "lb-spider",1136 "addition_date": "2012/05/07",1137 "instances": []1138 }1139 ,1140 {1141 "pattern": "Sogou",1142 "addition_date": "2012/05/13",1143 "url": "http://www.sogou.com/docs/help/webmasters.htm#07",1144 "instances": [1145 "Sogou News Spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)",1146 "Sogou Pic Spider/3.0(+http://www.sogou.com/docs/help/webmasters.htm#07)",1147 "Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)"1148 ]1149 }1150 ,1151 {1152 "pattern": "lssbot",1153 "addition_date": "2012/05/15",1154 "instances": []1155 }1156 ,1157 {1158 "pattern": "careerbot",1159 "addition_date": "2012/05/23",1160 "url": "http://www.career-x.de/bot.html",1161 "instances": []1162 }1163 ,1164 {1165 "pattern": "wotbox",1166 "addition_date": "2012/06/12",1167 "url": "http://www.wotbox.com",1168 "instances": [1169 "Wotbox/2.0 (bot@wotbox.com; http://www.wotbox.com)",1170 "Wotbox/2.01 (+http://www.wotbox.com/bot/)"1171 ]1172 }1173 ,1174 {1175 "pattern": "wocbot",1176 "addition_date": "2012/07/25",1177 "url": "http://www.wocodi.com/crawler",1178 "instances": []1179 }1180 ,1181 {1182 "pattern": "ichiro",1183 "addition_date": "2012/08/28",1184 "url": "http://help.goo.ne.jp/help/article/1142",1185 "instances": [1186 "DoCoMo/2.0 P900i(c100;TB;W24H11) (compatible; ichiro/mobile goo; +http://help.goo.ne.jp/help/article/1142/)",1187 "DoCoMo/2.0 P900i(c100;TB;W24H11) (compatible; ichiro/mobile goo; +http://search.goo.ne.jp/option/use/sub4/sub4-1/)",1188 "DoCoMo/2.0 P900i(c100;TB;W24H11) (compatible; ichiro/mobile goo;+http://search.goo.ne.jp/option/use/sub4/sub4-1/)",1189 "DoCoMo/2.0 P900i(c100;TB;W24H11)(compatible; ichiro/mobile goo;+http://help.goo.ne.jp/door/crawler.html)",1190 "DoCoMo/2.0 P901i(c100;TB;W24H11) (compatible; ichiro/mobile goo; +http://help.goo.ne.jp/door/crawler.html)",1191 "KDDI-CA31 UP.Browser/6.2.0.7.3.129 (GUI) MMP/2.0 (compatible; ichiro/mobile goo; +http://help.goo.ne.jp/help/article/1142/)",1192 "KDDI-CA31 UP.Browser/6.2.0.7.3.129 (GUI) MMP/2.0 (compatible; ichiro/mobile goo; +http://search.goo.ne.jp/option/use/sub4/sub4-1/)",1193 "KDDI-CA31 UP.Browser/6.2.0.7.3.129 (GUI) MMP/2.0 (compatible; ichiro/mobile goo;+http://search.goo.ne.jp/option/use/sub4/sub4-1/)",1194 "ichiro/2.0 (http://help.goo.ne.jp/door/crawler.html)",1195 "ichiro/2.0 (ichiro@nttr.co.jp)",1196 "ichiro/3.0 (http://help.goo.ne.jp/door/crawler.html)",1197 "ichiro/3.0 (http://help.goo.ne.jp/help/article/1142)",1198 "ichiro/3.0 (http://search.goo.ne.jp/option/use/sub4/sub4-1/)",1199 "ichiro/4.0 (http://help.goo.ne.jp/door/crawler.html)",1200 "ichiro/5.0 (http://help.goo.ne.jp/door/crawler.html)"1201 ]1202 }1203 ,1204 {1205 "pattern": "DuckDuckBot",1206 "addition_date": "2012/09/19",1207 "url": "http://duckduckgo.com/duckduckbot.html",1208 "instances": [1209 "DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)",1210 "DuckDuckBot/1.1; (+http://duckduckgo.com/duckduckbot.html)",1211 "Mozilla/5.0 (compatible; DuckDuckBot-Https/1.1; https://duckduckgo.com/duckduckbot)",1212 "'Mozilla/5.0 (compatible; DuckDuckBot-Https/1.1; https://duckduckgo.com/duckduckbot)'"1213 ]1214 }1215 ,1216 {1217 "pattern": "lssrocketcrawler",1218 "addition_date": "2012/09/24",1219 "instances": []1220 }1221 ,1222 {1223 "pattern": "drupact",1224 "addition_date": "2012/09/27",1225 "url": "http://www.arocom.de/drupact",1226 "instances": [1227 "drupact/0.7; http://www.arocom.de/drupact"1228 ]1229 }1230 ,1231 {1232 "pattern": "webcompanycrawler",1233 "addition_date": "2012/10/03",1234 "instances": []1235 }1236 ,1237 {1238 "pattern": "acoonbot",1239 "addition_date": "2012/10/07",1240 "url": "http://www.acoon.de/robot.asp",1241 "instances": []1242 }1243 ,1244 {1245 "pattern": "openindexspider",1246 "addition_date": "2012/10/26",1247 "url": "http://www.openindex.io/en/webmasters/spider.html",1248 "instances": []1249 }1250 ,1251 {1252 "pattern": "gnam gnam spider",1253 "addition_date": "2012/10/31",1254 "instances": []1255 }1256 ,1257 {1258 "pattern": "web-archive-net.com.bot",1259 "instances": []1260 }1261 ,1262 {1263 "pattern": "backlinkcrawler",1264 "addition_date": "2013/01/04",1265 "instances": []1266 }1267 ,1268 {1269 "pattern": "coccoc",1270 "addition_date": "2013/01/04",1271 "url": "http://help.coccoc.vn/",1272 "instances": [1273 "Mozilla/5.0 (compatible; coccoc/1.0; +http://help.coccoc.com/)",1274 "Mozilla/5.0 (compatible; coccoc/1.0; +http://help.coccoc.com/searchengine)",1275 "Mozilla/5.0 (compatible; coccocbot-image/1.0; +http://help.coccoc.com/searchengine)",1276 "Mozilla/5.0 (compatible; coccocbot-web/1.0; +http://help.coccoc.com/searchengine)",1277 "Mozilla/5.0 (compatible; image.coccoc/1.0; +http://help.coccoc.com/)",1278 "Mozilla/5.0 (compatible; imagecoccoc/1.0; +http://help.coccoc.com/)",1279 "Mozilla/5.0 (compatible; imagecoccoc/1.0; +http://help.coccoc.com/searchengine)",1280 "coccoc",1281 "coccoc/1.0 ()",1282 "coccoc/1.0 (http://help.coccoc.com/)",1283 "coccoc/1.0 (http://help.coccoc.vn/)"1284 ]1285 }1286 ,1287 {1288 "pattern": "integromedb",1289 "addition_date": "2013/01/10",1290 "url": "http://www.integromedb.org/Crawler",1291 "instances": [1292 "www.integromedb.org/Crawler"1293 ]1294 }1295 ,1296 {1297 "pattern": "content crawler spider",1298 "addition_date": "2013/01/11",1299 "instances": []1300 }1301 ,1302 {1303 "pattern": "toplistbot",1304 "addition_date": "2013/02/05",1305 "instances": []1306 }1307 ,1308 {1309 "pattern": "it2media-domain-crawler",1310 "addition_date": "2013/03/12",1311 "instances": [1312 "it2media-domain-crawler/1.0 on crawler-prod.it2media.de",1313 "it2media-domain-crawler/2.0"1314 ]1315 }1316 ,1317 {1318 "pattern": "ip-web-crawler.com",1319 "addition_date": "2013/03/22",1320 "instances": []1321 }1322 ,1323 {1324 "pattern": "siteexplorer.info",1325 "addition_date": "2013/05/01",1326 "instances": [1327 "Mozilla/5.0 (compatible; SiteExplorer/1.0b; +http://siteexplorer.info/)",1328 "Mozilla/5.0 (compatible; SiteExplorer/1.1b; +http://siteexplorer.info/Backlink-Checker-Spider/)"1329 ]1330 }1331 ,1332 {1333 "pattern": "elisabot",1334 "addition_date": "2013/06/27",1335 "instances": []1336 }1337 ,1338 {1339 "pattern": "proximic",1340 "addition_date": "2013/09/12",1341 "url": "http://www.proximic.com/info/spider.php",1342 "instances": [1343 "Mozilla/5.0 (compatible; proximic; +http://www.proximic.com)",1344 "Mozilla/5.0 (compatible; proximic; +http://www.proximic.com/info/spider.php)"1345 ]1346 }1347 ,1348 {1349 "pattern": "changedetection",1350 "addition_date": "2013/09/13",1351 "url": "http://www.changedetection.com/bot.html",1352 "instances": [1353 "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; http://www.changedetection.com/bot.html )"1354 ]1355 }1356 ,1357 {1358 "pattern": "arabot",1359 "addition_date": "2013/10/09",1360 "instances": []1361 }1362 ,1363 {1364 "pattern": "WeSEE:Search",1365 "addition_date": "2013/11/18",1366 "instances": [1367 "WeSEE:Search",1368 "WeSEE:Search/0.1 (Alpha, http://www.wesee.com/en/support/bot/)"1369 ]1370 }1371 ,1372 {1373 "pattern": "niki-bot",1374 "addition_date": "2014/01/01",1375 "instances": []1376 }1377 ,1378 {1379 "pattern": "CrystalSemanticsBot",1380 "addition_date": "2014/02/17",1381 "url": "http://www.crystalsemantics.com/user-agent/",1382 "instances": []1383 }1384 ,1385 {1386 "pattern": "rogerbot",1387 "addition_date": "2014/02/28",1388 "url": "http://moz.com/help/pro/what-is-rogerbot-",1389 "instances": [1390 "Mozilla/5.0 (compatible; rogerBot/1.0; UrlCrawler; http://www.seomoz.org/dp/rogerbot)",1391 "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+partager@moz.com)",1392 "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+shiny@moz.com)",1393 "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-wherecat@moz.com",1394 "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-wherecat@moz.com)",1395 "rogerbot/1.0 (http://www.moz.com/dp/rogerbot, rogerbot-crawler@moz.com)",1396 "rogerbot/1.0 (http://www.seomoz.org/dp/rogerbot, rogerbot-crawler+shiny@seomoz.org)",1397 "rogerbot/1.0 (http://www.seomoz.org/dp/rogerbot, rogerbot-crawler@seomoz.org)",1398 "rogerbot/1.0 (http://www.seomoz.org/dp/rogerbot, rogerbot-wherecat@moz.com)",1399 "rogerbot/1.1 (http://moz.com/help/guides/search-overview/crawl-diagnostics#more-help, rogerbot-crawler+pr2-crawler-05@moz.com)",1400 "rogerbot/1.1 (http://moz.com/help/guides/search-overview/crawl-diagnostics#more-help, rogerbot-crawler+pr4-crawler-11@moz.com)",1401 "rogerbot/1.1 (http://moz.com/help/guides/search-overview/crawl-diagnostics#more-help, rogerbot-crawler+pr4-crawler-15@moz.com)",1402 "rogerbot/1.2 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+phaser-testing-crawler-01@moz.com)"1403 ]1404 }1405 ,1406 {1407 "pattern": "360Spider",1408 "addition_date": "2014/03/14",1409 "url": "http://needs-be.blogspot.co.uk/2013/02/how-to-block-spider360.html",1410 "instances": [1411 "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1; 360Spider",1412 "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1; 360Spider(compatible; HaosouSpider; http://www.haosou.com/help/help_3_2.html)",1413 "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36 QIHU 360SE; 360Spider",1414 "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; ) Firefox/1.5.0.11; 360Spider",1415 "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.8.0.11) Firefox/1.5.0.11; 360Spider",1416 "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.8.0.11) Firefox/1.5.0.11 360Spider;",1417 "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.8.0.11) Gecko/20070312 Firefox/1.5.0.11; 360Spider",1418 "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider",1419 "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider(compatible; HaosouSpider; http://www.haosou.com/help/help_3_2.html)",1420 "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36; 360Spider"1421 ]1422 }1423 ,1424 {1425 "pattern": "psbot",1426 "addition_date": "2014/03/31",1427 "url": "http://www.picsearch.com/bot.html",1428 "instances": [1429 "psbot-image (+http://www.picsearch.com/bot.html)",1430 "psbot-page (+http://www.picsearch.com/bot.html)",1431 "psbot/0.1 (+http://www.picsearch.com/bot.html)"1432 ]1433 }1434 ,1435 {1436 "pattern": "InterfaxScanBot",1437 "addition_date": "2014/03/31",1438 "url": "http://scan-interfax.ru",1439 "instances": []1440 }1441 ,1442 {1443 "pattern": "CC Metadata Scaper",1444 "addition_date": "2014/04/01",1445 "url": "http://wiki.creativecommons.org/Metadata_Scraper",1446 "instances": [1447 "CC Metadata Scaper http://wiki.creativecommons.org/Metadata_Scraper"1448 ]1449 }1450 ,1451 {1452 "pattern": "g00g1e.net",1453 "addition_date": "2014/04/01",1454 "url": "http://www.g00g1e.net/",1455 "instances": []1456 }1457 ,1458 {1459 "pattern": "GrapeshotCrawler",1460 "addition_date": "2014/04/01",1461 "url": "http://www.grapeshot.co.uk/crawler.php",1462 "instances": [1463 "Mozilla/5.0 (compatible; GrapeshotCrawler/2.0; +http://www.grapeshot.co.uk/crawler.php)"1464 ]1465 }1466 ,1467 {1468 "pattern": "urlappendbot",1469 "addition_date": "2014/05/10",1470 "url": "http://www.profound.net/urlappendbot.html",1471 "instances": [1472 "Mozilla/5.0 (compatible; URLAppendBot/1.0; +http://www.profound.net/urlappendbot.html)"1473 ]1474 }1475 ,1476 {1477 "pattern": "brainobot",1478 "addition_date": "2014/06/24",1479 "instances": []1480 }1481 ,1482 {1483 "pattern": "fr-crawler",1484 "addition_date": "2014/07/31",1485 "instances": [1486 "Mozilla/5.0 (compatible; fr-crawler/1.1)"1487 ]1488 }1489 ,1490 {1491 "pattern": "binlar",1492 "addition_date": "2014/09/12",1493 "instances": [1494 "binlar_2.6.3 binlar2.6.3@unspecified.mail",1495 "binlar_2.6.3 binlar_2.6.3@unspecified.mail",1496 "binlar_2.6.3 larbin2.6.3@unspecified.mail",1497 "binlar_2.6.3 phanendra_kalapala@McAfee.com",1498 "binlar_2.6.3 test@mgmt.mic"1499 ]1500 }1501 ,1502 {1503 "pattern": "SimpleCrawler",1504 "addition_date": "2014/09/12",1505 "instances": [1506 "SimpleCrawler/0.1"1507 ]1508 }1509 ,1510 {1511 "pattern": "Twitterbot",1512 "addition_date": "2014/09/12",1513 "url": "https://dev.twitter.com/cards/getting-started",1514 "instances": [1515 "Twitterbot/0.1",1516 "Twitterbot/1.0"1517 ]1518 }1519 ,1520 {1521 "pattern": "cXensebot",1522 "addition_date": "2014/10/05",1523 "instances": [1524 "cXensebot/1.1a"1525 ],1526 "url": "http://www.cxense.com/bot.html"1527 }1528 ,1529 {1530 "pattern": "smtbot",1531 "addition_date": "2014/10/04",1532 "instances": [1533 "Mozilla/5.0 (compatible; SMTBot/1.0; +http://www.similartech.com/smtbot)",1534 "SMTBot (similartech.com/smtbot)",1535 "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; SMTBot/1.0; +http://www.similartech.com/smtbot)",1536 "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36 (compatible; SMTBot/1.0; +http://www.similartech.com/smtbot)",1537 "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36 (compatible; SMTBot/1.0; http://www.similartech.com/smtbot)"1538 ],1539 "url": "http://www.similartech.com/smtbot"1540 }1541 ,1542 {1543 "pattern": "bnf.fr_bot",1544 "addition_date": "2014/11/18",1545 "url": "http://www.bnf.fr/fr/outils/a.dl_web_capture_robot.html",1546 "instances": [1547 "Mozilla/5.0 (compatible; bnf.fr_bot; +http://bibnum.bnf.fr/robot/bnf.html)",1548 "Mozilla/5.0 (compatible; bnf.fr_bot; +http://www.bnf.fr/fr/outils/a.dl_web_capture_robot.html)"1549 ]1550 }1551 ,1552 {1553 "pattern": "A6-Indexer",1554 "addition_date": "2014/12/05",1555 "url": "http://www.a6corp.com/a6-web-scraping-policy/",1556 "instances": [1557 "A6-Indexer"1558 ]1559 }1560 ,1561 {1562 "pattern": "ADmantX",1563 "addition_date": "2014/12/05",1564 "url": "http://www.admantx.com",1565 "instances": [1566 "ADmantX Platform Semantic Analyzer - ADmantX Inc. - www.admantx.com - support@admantx.com"1567 ]1568 }1569 ,1570 {1571 "pattern": "Facebot",1572 "url": "https://developers.facebook.com/docs/sharing/best-practices#crawl",1573 "addition_date": "2014/12/30",1574 "instances": [1575 "Facebot/1.0"1576 ]1577 }1578 ,1579 {1580 "pattern": "OrangeBot\\/",1581 "instances": [1582 "Mozilla/5.0 (compatible; OrangeBot/2.0; support.orangebot@orange.com"1583 ],1584 "addition_date": "2015/01/12"1585 }1586 ,1587 {1588 "pattern": "memorybot",1589 "url": "http://mignify.com/bot.htm",1590 "instances": [1591 "Mozilla/5.0 (compatible; memorybot/1.21.14 +http://mignify.com/bot.html)"1592 ],1593 "addition_date": "2015/02/01"1594 }1595 ,1596 {1597 "pattern": "AdvBot",1598 "url": "http://advbot.net/bot.html",1599 "instances": [1600 "Mozilla/5.0 (compatible; AdvBot/2.0; +http://advbot.net/bot.html)"1601 ],1602 "addition_date": "2015/02/01"1603 }1604 ,1605 {1606 "pattern": "MegaIndex",1607 "url": "https://www.megaindex.ru/?tab=linkAnalyze",1608 "instances": [1609 "Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +https://www.megaindex.ru/?tab=linkAnalyze)",1610 "Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +http://megaindex.com/crawler)"1611 ],1612 "addition_date": "2015/03/28"1613 }1614 ,1615 {1616 "pattern": "SemanticScholarBot",1617 "url": "https://www.semanticscholar.org/crawler",1618 "instances": [1619 "SemanticScholarBot/1.0 (+http://s2.allenai.org/bot.html)",1620 "Mozilla/5.0 (compatible) SemanticScholarBot (+https://www.semanticscholar.org/crawler)"1621 ],1622 "addition_date": "2015/03/28"1623 }1624 ,1625 {1626 "pattern": "ltx71",1627 "url": "http://ltx71.com/",1628 "instances": [1629 "ltx71 - (http://ltx71.com/)"1630 ],1631 "addition_date": "2015/04/04"1632 }1633 ,1634 {1635 "pattern": "nerdybot",1636 "url": "http://nerdybot.com/",1637 "instances": [1638 "nerdybot"1639 ],1640 "addition_date": "2015/04/05"1641 }1642 ,1643 {1644 "pattern": "xovibot",1645 "url": "http://www.xovibot.net/",1646 "instances": [1647 "Mozilla/5.0 (compatible; XoviBot/2.0; +http://www.xovibot.net/)"1648 ],1649 "addition_date": "2015/04/05"1650 }1651 ,1652 {1653 "pattern": "BUbiNG",1654 "url": "http://law.di.unimi.it/BUbiNG.html",1655 "instances": [1656 "BUbiNG (+http://law.di.unimi.it/BUbiNG.html)"1657 ],1658 "addition_date": "2015/04/06"1659 }1660 ,1661 {1662 "pattern": "Qwantify",1663 "url": "https://www.qwant.com/",1664 "instances": [1665 "Mozilla/5.0 (compatible; Qwantify/2.0n; +https://www.qwant.com/)/*",1666 "Mozilla/5.0 (compatible; Qwantify/2.4w; +https://www.qwant.com/)/2.4w",1667 "Mozilla/5.0 (compatible; Qwantify/Bleriot/1.1; +https://help.qwant.com/bot)",1668 "Mozilla/5.0 (compatible; Qwantify/Bleriot/1.2.1; +https://help.qwant.com/bot)"1669 ],1670 "addition_date": "2015/04/06"1671 }1672 ,1673 {1674 "pattern": "archive.org_bot",1675 "url": "http://www.archive.org/details/archive.org_bot",1676 "depends_on": ["heritrix"],1677 "instances": [1678 "Mozilla/5.0 (compatible; heritrix/3.1.1-SNAPSHOT-20120116.200628 +http://www.archive.org/details/archive.org_bot)",1679 "Mozilla/5.0 (compatible; archive.org_bot/heritrix-1.15.4 +http://www.archive.org)",1680 "Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-20140702-2247 +http://archive.org/details/archive.org_bot)",1681 "Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot)",1682 "Mozilla/5.0 (compatible; archive.org_bot +http://archive.org/details/archive.org_bot)",1683 "Mozilla/5.0 (compatible; special_archiver/3.1.1 +http://www.archive.org/details/archive.org_bot)"1684 ],1685 "addition_date": "2015/04/14"1686 }1687 ,1688 {1689 "pattern": "Applebot",1690 "url": "http://www.apple.com/go/applebot",1691 "addition_date": "2015/04/15",1692 "instances": [1693 "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1)",1694 "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1; +http://www.apple.com/go/applebot)",1695 "Mozilla/5.0 (compatible; Applebot/0.3; +http://www.apple.com/go/applebot)",1696 "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Applebot/0.3; +http://www.apple.com/go/applebot)",1697 "Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B410 Safari/600.1.4 (Applebot/0.1; +http://www.apple.com/go/applebot)"1698 ]1699 }1700 ,1701 {1702 "pattern": "TweetmemeBot",1703 "url": "http://datasift.com/bot.html",1704 "instances": [1705 "Mozilla/5.0 (TweetmemeBot/4.0; +http://datasift.com/bot.html) Gecko/20100101 Firefox/31.0"1706 ],1707 "addition_date": "2015/04/15"1708 }1709 ,1710 {1711 "pattern": "crawler4j",1712 "url": "https://github.com/yasserg/crawler4j",1713 "instances": [1714 "crawler4j (http://code.google.com/p/crawler4j/)",1715 "crawler4j (https://github.com/yasserg/crawler4j/)"1716 ],1717 "addition_date": "2015/05/07"1718 }1719 ,1720 {1721 "pattern": "findxbot",1722 "url": "http://www.findxbot.com",1723 "instances": [1724 "Mozilla/5.0 (compatible; Findxbot/1.0; +http://www.findxbot.com)"1725 ],1726 "addition_date": "2015/05/07"1727 }1728 ,1729 {1730 "pattern": "S[eE][mM]rushBot",1731 "url": "http://www.semrush.com/bot.html",1732 "instances": [1733 "Mozilla/5.0 (compatible; SemrushBot-SA/0.97; +http://www.semrush.com/bot.html)",1734 "Mozilla/5.0 (compatible; SemrushBot-SI/0.97; +http://www.semrush.com/bot.html)",1735 "Mozilla/5.0 (compatible; SemrushBot/3~bl; +http://www.semrush.com/bot.html)",1736 "Mozilla/5.0 (compatible; SemrushBot/0.98~bl; +http://www.semrush.com/bot.html)",1737 "Mozilla/5.0 (compatible; SemrushBot-BA; +http://www.semrush.com/bot.html)",1738 "Mozilla/5.0 (compatible; SemrushBot/6~bl; +http://www.semrush.com/bot.html)",1739 "SEMrushBot"1740 ],1741 "addition_date": "2015/05/26"1742 }1743 ,1744 {1745 "pattern": "yoozBot",1746 "url": "http://yooz.ir",1747 "instances": [1748 "Mozilla/5.0 (compatible; yoozBot-2.2; http://yooz.ir; info@yooz.ir)"1749 ],1750 "addition_date": "2015/05/26"1751 }1752 ,1753 {1754 "pattern": "lipperhey",1755 "url": "http://www.lipperhey.com/",1756 "instances": [1757 "Mozilla/5.0 (compatible; Lipperhey Link Explorer; http://www.lipperhey.com/)",1758 "Mozilla/5.0 (compatible; Lipperhey SEO Service; http://www.lipperhey.com/)",1759 "Mozilla/5.0 (compatible; Lipperhey Site Explorer; http://www.lipperhey.com/)",1760 "Mozilla/5.0 (compatible; Lipperhey-Kaus-Australis/5.0; +https://www.lipperhey.com/en/about/)"1761 ],1762 "addition_date": "2015/08/26"1763 }1764 ,1765 {1766 "pattern": "Y!J",1767 "url": "https://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/~/%E3%82%A6%E3%82%A7%E3%83%96%E3%83%9A%E3%83%BC%E3%82%B8%E3%81%AB%E3%82%A2%E3%82%AF%E3%82%BB%E3%82%B9%E3%81%99%E3%82%8B%E3%82%B7%E3%82%B9%E3%83%86%E3%83%A0%E3%81%AE%E3%83%A6%E3%83%BC%E3%82%B6%E3%83%BC%E3%82%A8%E3%83%BC%E3%82%B8%E3%82%A7%E3%83%B3%E3%83%88%E3%81%AB%E3%81%A4%E3%81%84%E3%81%A6",1768 "instances": [1769 "Y!J-ASR/0.1 crawler (http://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/)",1770 "Y!J-BRJ/YATS crawler (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)",1771 "Y!J-PSC/1.0 crawler (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)",1772 "Y!J-BRW/1.0 crawler (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)",1773 "Mozilla/5.0 (iPhone; Y!J-BRY/YATSH crawler; http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)",1774 "Mozilla/5.0 (compatible; Y!J SearchMonkey/1.0 (Y!J-AGENT; http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html))"1775 ],1776 "addition_date": "2015/05/26"1777 }1778 ,1779 {1780 "pattern": "Domain Re-Animator Bot",1781 "url": "http://domainreanimator.com",1782 "instances": [1783 "Domain Re-Animator Bot (http://domainreanimator.com) - support@domainreanimator.com"1784 ],1785 "addition_date": "2015/04/14"1786 }1787 ,1788 {1789 "pattern": "AddThis",1790 "url": "https://www.addthis.com",1791 "instances": [1792 "AddThis.com robot tech.support@clearspring.com"1793 ],1794 "addition_date": "2015/06/02"1795 }1796 ,1797 {1798 "pattern": "Screaming Frog SEO Spider",1799 "url": "http://www.screamingfrog.co.uk/seo-spider",1800 "instances": [1801 "Screaming Frog SEO Spider/5.1"1802 ],1803 "addition_date": "2016/01/08"1804 }1805 ,1806 {1807 "pattern": "MetaURI",1808 "url": "http://www.useragentstring.com/MetaURI_id_17683.php",1809 "instances": [1810 "MetaURI API/2.0 +metauri.com"1811 ],1812 "addition_date": "2016/01/02"1813 }1814 ,1815 {1816 "pattern": "Scrapy",1817 "url": "http://scrapy.org/",1818 "instances": [1819 "Scrapy/1.0.3 (+http://scrapy.org)"1820 ],1821 "addition_date": "2016/01/02"1822 }1823 ,1824 {1825 "pattern": "Livelap[bB]ot",1826 "url": "http://site.livelap.com/crawler",1827 "instances": [1828 "LivelapBot/0.2 (http://site.livelap.com/crawler)",1829 "Livelapbot/0.1"1830 ],1831 "addition_date": "2016/01/02"1832 }1833 ,1834 {1835 "pattern": "OpenHoseBot",1836 "url": "http://www.openhose.org/bot.html",1837 "instances": [1838 "Mozilla/5.0 (compatible; OpenHoseBot/2.1; +http://www.openhose.org/bot.html)"1839 ],1840 "addition_date": "2016/01/02"1841 }1842 ,1843 {1844 "pattern": "CapsuleChecker",1845 "url": "http://www.capsulink.com/about",1846 "instances": [1847 "CapsuleChecker (http://www.capsulink.com/)"1848 ],1849 "addition_date": "2016/01/02"1850 }1851 ,1852 {1853 "pattern": "collection@infegy.com",1854 "url": "http://infegy.com/",1855 "instances": [1856 "Mozilla/5.0 (compatible) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.73 Safari/537.36 collection@infegy.com"1857 ],1858 "addition_date": "2016/01/03"1859 }1860 ,1861 {1862 "pattern": "IstellaBot",1863 "url": "http://www.tiscali.it/",1864 "instances": [1865 "Mozilla/5.0 (compatible; IstellaBot/1.23.15 +http://www.tiscali.it/)"1866 ],1867 "addition_date": "2016/01/09"1868 }1869 ,1870 {1871 "pattern": "DeuSu\\/",1872 "addition_date": "2016/01/23",1873 "url": "https://deusu.de/robot.html",1874 "instances": [1875 "Mozilla/5.0 (compatible; DeuSu/0.1.0; +https://deusu.org)",1876 "Mozilla/5.0 (compatible; DeuSu/5.0.2; +https://deusu.de/robot.html)"1877 ]1878 }1879 ,1880 {1881 "pattern": "betaBot",1882 "addition_date": "2016/01/23",1883 "instances": []1884 }1885 ,1886 {1887 "pattern": "Cliqzbot\\/",1888 "addition_date": "2016/01/23",1889 "url": "http://cliqz.com/company/cliqzbot",1890 "instances": [1891 "Mozilla/5.0 (compatible; Cliqzbot/2.0; +http://cliqz.com/company/cliqzbot)",1892 "Cliqzbot/0.1 (+http://cliqz.com +cliqzbot@cliqz.com)",1893 "Cliqzbot/0.1 (+http://cliqz.com/company/cliqzbot)",1894 "Mozilla/5.0 (compatible; Cliqzbot/0.1 +http://cliqz.com/company/cliqzbot)",1895 "Mozilla/5.0 (compatible; Cliqzbot/1.0 +http://cliqz.com/company/cliqzbot)"1896 ]1897 }1898 ,1899 {1900 "pattern": "MojeekBot\\/",1901 "addition_date": "2016/01/23",1902 "url": "https://www.mojeek.com/bot.html",1903 "instances": [1904 "MojeekBot/0.2 (archi; http://www.mojeek.com/bot.html)",1905 "Mozilla/5.0 (compatible; MojeekBot/0.2; http://www.mojeek.com/bot.html#relaunch)",1906 "Mozilla/5.0 (compatible; MojeekBot/0.2; http://www.mojeek.com/bot.html)",1907 "Mozilla/5.0 (compatible; MojeekBot/0.5; http://www.mojeek.com/bot.html)",1908 "Mozilla/5.0 (compatible; MojeekBot/0.6; +https://www.mojeek.com/bot.html)",1909 "Mozilla/5.0 (compatible; MojeekBot/0.6; http://www.mojeek.com/bot.html)"1910 ]1911 }1912 ,1913 {1914 "pattern": "netEstate NE Crawler",1915 "addition_date": "2016/01/23",1916 "url": "+http://www.website-datenbank.de/",1917 "instances": [1918 "netEstate NE Crawler (+http://www.sengine.info/)",1919 "netEstate NE Crawler (+http://www.website-datenbank.de/)"1920 ]1921 }1922 ,1923 {1924 "pattern": "SafeSearch microdata crawler",1925 "addition_date": "2016/01/23",1926 "url": "https://safesearch.avira.com",1927 "instances": [1928 "SafeSearch microdata crawler (https://safesearch.avira.com, safesearch-abuse@avira.com)"1929 ]1930 }1931 ,1932 {1933 "pattern": "Gluten Free Crawler\\/",1934 "addition_date": "2016/01/23",1935 "url": "http://glutenfreepleasure.com/",1936 "instances": [1937 "Mozilla/5.0 (compatible; Gluten Free Crawler/1.0; +http://glutenfreepleasure.com/)"1938 ]1939 }1940 ,1941 {1942 "pattern": "Sonic",1943 "addition_date": "2016/02/08",1944 "url": "http://www.yama.info.waseda.ac.jp/~crawler/info.html",1945 "instances": [1946 "Mozilla/5.0 (compatible; RankSonicSiteAuditor/1.0; +https://ranksonic.com/ranksonic_sab.html)",1947 "Mozilla/5.0 (compatible; Sonic/1.0; http://www.yama.info.waseda.ac.jp/~crawler/info.html)",1948 "Mozzila/5.0 (compatible; Sonic/1.0; http://www.yama.info.waseda.ac.jp/~crawler/info.html)"1949 ]1950 }1951 ,1952 {1953 "pattern": "Sysomos",1954 "addition_date": "2016/02/08",1955 "url": "http://www.sysomos.com",1956 "instances": [1957 "Mozilla/5.0 (compatible; Sysomos/1.0; +http://www.sysomos.com/; Sysomos)"1958 ]1959 }1960 ,1961 {1962 "pattern": "Trove",1963 "addition_date": "2016/02/08",1964 "url": "http://www.trove.com",1965 "instances": []1966 }1967 ,1968 {1969 "pattern": "deadlinkchecker",1970 "addition_date": "2016/02/08",1971 "url": "http://www.deadlinkchecker.com",1972 "instances": [1973 "www.deadlinkchecker.com Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36",1974 "www.deadlinkchecker.com XMLHTTP/1.0",1975 "www.deadlinkchecker.com XMLHTTP/1.0 Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36"1976 ]1977 }1978 ,1979 {1980 "pattern": "Slack-ImgProxy",1981 "addition_date": "2016/04/25",1982 "url": "https://api.slack.com/robots",1983 "instances": [1984 "Slack-ImgProxy (+https://api.slack.com/robots)",1985 "Slack-ImgProxy 0.59 (+https://api.slack.com/robots)",1986 "Slack-ImgProxy 0.66 (+https://api.slack.com/robots)",1987 "Slack-ImgProxy 1.106 (+https://api.slack.com/robots)",1988 "Slack-ImgProxy 1.138 (+https://api.slack.com/robots)",1989 "Slack-ImgProxy 149 (+https://api.slack.com/robots)"1990 ]1991 }1992 ,1993 {1994 "pattern": "Embedly",1995 "addition_date": "2016/04/25",1996 "url": "http://support.embed.ly",1997 "instances": [1998 "Embedly +support@embed.ly",1999 "Mozilla/5.0 (compatible; Embedly/0.2; +http://support.embed.ly/)",2000 "Mozilla/5.0 (compatible; Embedly/0.2; snap; +http://support.embed.ly/)"2001 ]2002 }2003 ,2004 {2005 "pattern": "RankActiveLinkBot",2006 "addition_date": "2016/06/20",2007 "url": "https://rankactive.com/resources/rankactive-linkbot",2008 "instances": [2009 "Mozilla/5.0 (compatible; RankActiveLinkBot; +https://rankactive.com/resources/rankactive-linkbot)"2010 ]2011 }2012 ,2013 {2014 "pattern": "iskanie",2015 "addition_date": "2016/09/02",2016 "url": "http://www.iskanie.com",2017 "instances": [2018 "iskanie (+http://www.iskanie.com)"2019 ]2020 }2021 ,2022 {2023 "pattern": "SafeDNSBot",2024 "addition_date": "2016/09/10",2025 "url": "https://www.safedns.com/searchbot",2026 "instances": [2027 "SafeDNSBot (https://www.safedns.com/searchbot)"2028 ]2029 }2030 ,2031 {2032 "pattern": "SkypeUriPreview",2033 "addition_date": "2016/10/10",2034 "instances": [2035 "Mozilla/5.0 (Windows NT 6.1; WOW64) SkypeUriPreview Preview/0.5"2036 ]2037 }2038 ,2039 {2040 "pattern": "Veoozbot",2041 "addition_date": "2016/11/03",2042 "url": "http://www.veooz.com/veoozbot.html",2043 "instances": [2044 "Mozilla/5.0 (compatible; Veoozbot/1.0; +http://www.veooz.com/veoozbot.html)"2045 ]2046 }2047 ,2048 {2049 "pattern": "Slackbot",2050 "addition_date": "2016/11/03",2051 "url": "https://api.slack.com/robots",2052 "instances": [2053 "Slackbot-LinkExpanding (+https://api.slack.com/robots)",2054 "Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)",2055 "Slackbot 1.0 (+https://api.slack.com/robots)"2056 ]2057 }2058 ,2059 {2060 "pattern": "redditbot",2061 "addition_date": "2016/11/03",2062 "url": "http://www.reddit.com/feedback",2063 "instances": [2064 "Mozilla/5.0 (compatible; redditbot/1.0; +http://www.reddit.com/feedback)"2065 ]2066 }2067 ,2068 {2069 "pattern": "datagnionbot",2070 "addition_date": "2016/11/03",2071 "url": "http://www.datagnion.com/bot.html",2072 "instances": [2073 "datagnionbot (+http://www.datagnion.com/bot.html)"2074 ]2075 }2076 ,2077 {2078 "pattern": "Google-Adwords-Instant",2079 "addition_date": "2016/11/03",2080 "url": "http://www.google.com/adsbot.html",2081 "instances": [2082 "Google-Adwords-Instant (+http://www.google.com/adsbot.html)"2083 ]2084 }2085 ,2086 {2087 "pattern": "adbeat_bot",2088 "addition_date": "2016/11/04",2089 "instances": [2090 "Mozilla/5.0 (compatible; adbeat_bot; +support@adbeat.com; support@adbeat.com)",2091 "adbeat_bot"2092 ]2093 }2094 ,2095 {2096 "pattern": "WhatsApp",2097 "addition_date": "2016/11/15",2098 "url": "https://www.whatsapp.com/",2099 "instances": [2100 "WhatsApp",2101 "WhatsApp/0.3.4479 N",2102 "WhatsApp/0.3.4679 N",2103 "WhatsApp/0.3.4941 N",2104 "WhatsApp/2.12.15/i",2105 "WhatsApp/2.12.16/i",2106 "WhatsApp/2.12.17/i",2107 "WhatsApp/2.12.449 A",2108 "WhatsApp/2.12.453 A",2109 "WhatsApp/2.12.510 A",2110 "WhatsApp/2.12.540 A",2111 "WhatsApp/2.12.548 A",2112 "WhatsApp/2.12.555 A",2113 "WhatsApp/2.12.556 A",2114 "WhatsApp/2.16.1/i",2115 "WhatsApp/2.16.13 A",2116 "WhatsApp/2.16.2/i",2117 "WhatsApp/2.16.42 A",2118 "WhatsApp/2.16.57 A",2119 "WhatsApp/2.19.92 i",2120 "WhatsApp/2.19.175 A",2121 "WhatsApp/2.19.244 A",2122 "WhatsApp/2.19.258 A",2123 "WhatsApp/2.19.308 A",2124 "WhatsApp/2.19.330 A"2125 ]2126 }2127 ,2128 {2129 "pattern": "contxbot",2130 "addition_date": "2017/02/25",2131 "instances": [2132 "Mozilla/5.0 (compatible;contxbot/1.0)"2133 ]2134 }2135 ,2136 {2137 "pattern": "pinterest.com.bot",2138 "addition_date": "2017/03/03",2139 "instances": [2140 "Mozilla/5.0 (compatible; Pinterestbot/1.0; +http://www.pinterest.com/bot.html)",2141 "Pinterest/0.2 (+http://www.pinterest.com/bot.html)"2142 ],2143 "url": "http://www.pinterest.com/bot.html"2144 }2145 ,2146 {2147 "pattern": "electricmonk",2148 "addition_date": "2017/03/04",2149 "instances": [2150 "Mozilla/5.0 (compatible; electricmonk/3.2.0 +https://www.duedil.com/our-crawler/)"2151 ],2152 "url": "https://www.duedil.com/our-crawler/"2153 }2154 ,2155 {2156 "pattern": "GarlikCrawler",2157 "addition_date": "2017/03/18",2158 "instances": [2159 "GarlikCrawler/1.2 (http://garlik.com/, crawler@garlik.com)"2160 ],2161 "url": "http://garlik.com/"2162 }2163 ,2164 {2165 "pattern": "BingPreview\\/",2166 "addition_date": "2017/04/23",2167 "url": "https://www.bing.com/webmaster/help/which-crawlers-does-bing-use-8c184ec0",2168 "instances": [2169 "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534+ (KHTML, like Gecko) BingPreview/1.0b",2170 "Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0; BingPreview/1.0b) like Gecko",2171 "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0; WOW64; Trident/6.0; BingPreview/1.0b)",2172 "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; WOW64; Trident/5.0; BingPreview/1.0b)",2173 "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 BingPreview/1.0b"2174 ]2175 }2176 ,2177 {2178 "pattern": "vebidoobot",2179 "addition_date": "2017/05/08",2180 "instances": [2181 "Mozilla/5.0 (compatible; vebidoobot/1.0; +https://blog.vebidoo.de/vebidoobot/"2182 ],2183 "url": "https://blog.vebidoo.de/vebidoobot/"2184 }2185 ,2186 {2187 "pattern": "FemtosearchBot",2188 "addition_date": "2017/05/16",2189 "instances": [2190 "Mozilla/5.0 (compatible; FemtosearchBot/1.0; http://femtosearch.com)"2191 ],2192 "url": "http://femtosearch.com"2193 }2194 ,2195 {2196 "pattern": "Yahoo Link Preview",2197 "addition_date": "2017/06/28",2198 "instances": [2199 "Mozilla/5.0 (compatible; Yahoo Link Preview; https://help.yahoo.com/kb/mail/yahoo-link-preview-SLN23615.html)"2200 ],2201 "url": "https://help.yahoo.com/kb/mail/yahoo-link-preview-SLN23615.html"2202 }2203 ,2204 {2205 "pattern": "MetaJobBot",2206 "addition_date": "2017/08/16",2207 "instances": [2208 "Mozilla/5.0 (compatible; MetaJobBot; http://www.metajob.de/crawler)"2209 ],2210 "url": "http://www.metajob.de/the/crawler"2211 }2212 ,2213 {2214 "pattern": "DomainStatsBot",2215 "addition_date": "2017/08/16",2216 "instances": [2217 "DomainStatsBot/1.0 (http://domainstats.io/our-bot)"2218 ],2219 "url": "http://domainstats.io/our-bot"2220 }2221 ,2222 {2223 "pattern": "mindUpBot",2224 "addition_date": "2017/08/16",2225 "instances": [2226 "mindUpBot (datenbutler.de)"2227 ],2228 "url": "http://www.datenbutler.de/"2229 }2230 ,2231 {2232 "pattern": "Daum\\/",2233 "addition_date": "2017/08/16",2234 "instances": [2235 "Mozilla/5.0 (compatible; Daum/4.1; +http://cs.daum.net/faq/15/4118.html?faqId=28966)"2236 ],2237 "url": "http://cs.daum.net/faq/15/4118.html?faqId=28966"2238 }2239 ,2240 {2241 "pattern": "Jugendschutzprogramm-Crawler",2242 "addition_date": "2017/08/16",2243 "instances": [2244 "Jugendschutzprogramm-Crawler; Info: http://www.jugendschutzprogramm.de"2245 ],2246 "url": "http://www.jugendschutzprogramm.de"2247 }2248 ,2249 {2250 "pattern": "Xenu Link Sleuth",2251 "addition_date": "2017/08/19",2252 "instances": [2253 "Xenu Link Sleuth/1.3.8"2254 ],2255 "url": "http://home.snafu.de/tilman/xenulink.html"2256 }2257 ,2258 {2259 "pattern": "Pcore-HTTP",2260 "addition_date": "2017/08/19",2261 "instances": [2262 "Pcore-HTTP/v0.40.3",2263 "Pcore-HTTP/v0.44.0"2264 ],2265 "url": "https://bitbucket.org/softvisio/pcore/overview"2266 }2267 ,2268 {2269 "pattern": "moatbot",2270 "addition_date": "2017/09/16",2271 "instances": [2272 "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.111 Safari/537.36 moatbot",2273 "Mozilla/5.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/600.1.3 (KHTML, like Gecko) Version/8.0 Mobile/12A4345d Safari/600.1.4 moatbot"2274 ],2275 "url": "https://moat.com"2276 }2277 ,2278 {2279 "pattern": "KosmioBot",2280 "addition_date": "2017/09/16",2281 "instances": [2282 "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.125 Safari/537.36 (compatible; KosmioBot/1.0; +http://kosm.io/bot.html)"2283 ],2284 "url": "http://kosm.io/bot.html"2285 }2286 ,2287 {2288 "pattern": "pingdom",2289 "addition_date": "2017/09/16",2290 "instances": [2291 "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/59.0.3071.109 Chrome/59.0.3071.109 Safari/537.36 PingdomPageSpeed/1.0 (pingbot/2.0; +http://www.pingdom.com/)",2292 "Mozilla/5.0 (compatible; pingbot/2.0; +http://www.pingdom.com/)"2293 ],2294 "url": "http://www.pingdom.com"2295 }2296 ,2297 {2298 "pattern": "AppInsights",2299 "addition_date": "2019/03/09",2300 "instances": [2301 "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; AppInsights)"2302 ],2303 "url": "https://docs.microsoft.com/en-us/azure/azure-monitor/app/app-insights-overview"2304 }2305 ,2306 {2307 "pattern": "PhantomJS",2308 "addition_date": "2017/09/18",2309 "instances": [2310 "Mozilla/5.0 (Unknown; Linux x86_64) AppleWebKit/538.1 (KHTML, like Gecko) PhantomJS/2.1.1 Safari/538.1 bl.uk_lddc_renderbot/2.0.0 (+ http://www.bl.uk/aboutus/legaldeposit/websites/websites/faqswebmaster/index.html)"2311 ],2312 "url": "http://phantomjs.org/"2313 }2314 ,2315 {2316 "pattern": "Gowikibot",2317 "addition_date": "2017/10/26",2318 "instances": [2319 "Mozilla/5.0 (compatible; Gowikibot/1.0; +http://www.gowikibot.com)"2320 ],2321 "url": "http://www.gowikibot.com"2322 }2323 ,2324 {2325 "pattern": "PiplBot",2326 "addition_date": "2017/10/30",2327 "instances": [2328 "PiplBot (+http://www.pipl.com/bot/)",2329 "Mozilla/5.0+(compatible;+PiplBot;+http://www.pipl.com/bot/)"2330 ],2331 "url": "http://www.pipl.com/bot/"2332 }2333 ,2334 {2335 "pattern": "Discordbot",2336 "addition_date": "2017/09/22",2337 "url": "https://discordapp.com",2338 "instances": [2339 "Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)"2340 ]2341 }2342 ,2343 {2344 "pattern": "TelegramBot",2345 "addition_date": "2017/10/01",2346 "instances": [2347 "TelegramBot (like TwitterBot)"2348 ]2349 }2350 ,2351 {2352 "pattern": "Jetslide",2353 "addition_date": "2017/09/27",2354 "url": "http://jetsli.de/crawler",2355 "instances": [2356 "Mozilla/5.0 (compatible; Jetslide; +http://jetsli.de/crawler)"2357 ]2358 }2359 ,2360 {2361 "pattern": "newsharecounts",2362 "addition_date": "2017/09/30",2363 "url": "http://newsharecounts.com/crawler",2364 "instances": [2365 "Mozilla/5.0 (compatible; NewShareCounts.com/1.0; +http://newsharecounts.com/crawler)"2366 ]2367 }2368 ,2369 {2370 "pattern": "James BOT",2371 "addition_date": "2017/10/12",2372 "url": "http://cognitiveseo.com/bot.html",2373 "instances": [2374 "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6 - James BOT - WebCrawler http://cognitiveseo.com/bot.html"2375 ]2376 }2377 ,2378 {2379 "pattern": "Bark[rR]owler",2380 "addition_date": "2017/10/09",2381 "url": "http://www.exensa.com/crawl",2382 "instances": [2383 "Barkrowler/0.5.1 (experimenting / debugging - sorry for your logs ) http://www.exensa.com/crawl - admin@exensa.com -- based on BuBiNG",2384 "Barkrowler/0.7 (+http://www.exensa.com/crawl)",2385 "BarkRowler/0.7 (+http://www.exensa.com/crawling)",2386 "Barkrowler/0.9 (+http://www.exensa.com/crawl)"2387 ]2388 }2389 ,2390 {2391 "pattern": "TinEye",2392 "addition_date": "2017/10/14",2393 "url": "http://www.tineye.com/crawler.html",2394 "instances": [2395 "Mozilla/5.0 (compatible; TinEye-bot/1.31; +http://www.tineye.com/crawler.html)",2396 "TinEye/1.1 (http://tineye.com/crawler.html)"2397 ]2398 }2399 ,2400 {2401 "pattern": "SocialRankIOBot",2402 "addition_date": "2017/10/19",2403 "url": "http://socialrank.io/about",2404 "instances": [2405 "SocialRankIOBot; http://socialrank.io/about"2406 ]2407 }2408 ,2409 {2410 "pattern": "trendictionbot",2411 "addition_date": "2017/10/30",2412 "url": "http://www.trendiction.de/bot",2413 "instances": [2414 "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-GB; rv:1.0; trendictionbot0.5.0; trendiction search; http://www.trendiction.de/bot; please let us know of any problems; web at trendiction.com) Gecko/20071127 Firefox/3.0.0.11",2415 "Mozilla/5.0 (Windows NT 10.0; Win64; x64; trendictionbot0.5.0; trendiction search; http://www.trendiction.de/bot; please let us know of any problems; web at trendiction.com) Gecko/20170101 Firefox/67.0"2416 ]2417 }2418 ,2419 {2420 "pattern": "Ocarinabot",2421 "addition_date": "2017/09/27",2422 "instances": [2423 "Ocarinabot"2424 ]2425 }2426 ,2427 {2428 "pattern": "epicbot",2429 "addition_date": "2017/10/31",2430 "url": "http://www.epictions.com/epicbot",2431 "instances": [2432 "Mozilla/5.0 (compatible; epicbot; +http://www.epictions.com/epicbot)"2433 ]2434 }2435 ,2436 {2437 "pattern": "Primalbot",2438 "addition_date": "2017/09/27",2439 "url": "https://www.primal.com",2440 "instances": [2441 "Mozilla/5.0 (compatible; Primalbot; +https://www.primal.com;)"2442 ]2443 }2444 ,2445 {2446 "pattern": "DuckDuckGo-Favicons-Bot",2447 "addition_date": "2017/10/06",2448 "url": "http://duckduckgo.com",2449 "instances": [2450 "Mozilla/5.0 (compatible; DuckDuckGo-Favicons-Bot/1.0; +http://duckduckgo.com)"2451 ]2452 }2453 ,2454 {2455 "pattern": "GnowitNewsbot",2456 "addition_date": "2017/10/30",2457 "url": "http://www.gnowit.com",2458 "instances": [2459 "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:49.0) Gecko/20100101 Firefox/49.0 / GnowitNewsbot / Contact information at http://www.gnowit.com"2460 ]2461 }2462 ,2463 {2464 "pattern": "Leikibot",2465 "addition_date": "2017/09/24",2466 "url": "http://www.leiki.com",2467 "instances": [2468 "Mozilla/5.0 (Windows NT 6.3;compatible; Leikibot/1.0; +http://www.leiki.com)"2469 ]2470 }2471 ,2472 {2473 "pattern": "LinkArchiver",2474 "addition_date": "2017/09/24",2475 "instances": [2476 "@LinkArchiver twitter bot"2477 ]2478 }2479 ,2480 {2481 "pattern": "YaK\\/",2482 "addition_date": "2017/09/25",2483 "url": "http://linkfluence.com",2484 "instances": [2485 "Mozilla/5.0 (compatible; YaK/1.0; http://linkfluence.com/; bot@linkfluence.com)"2486 ]2487 }2488 ,2489 {2490 "pattern": "PaperLiBot",2491 "addition_date": "2017/09/25",2492 "url": "http://support.paper.li/entries/20023257-what-is-paper-li",2493 "instances": [2494 "Mozilla/5.0 (compatible; PaperLiBot/2.1; http://support.paper.li/entries/20023257-what-is-paper-li)",2495 "Mozilla/5.0 (compatible; PaperLiBot/2.1; https://support.paper.li/entries/20023257-what-is-paper-li)"2496 2497 ]2498 }2499 ,2500 {2501 "pattern": "Digg Deeper",2502 "addition_date": "2017/09/26",2503 "url": "http://digg.com/about",2504 "instances": [2505 "Digg Deeper/v1 (http://digg.com/about)"2506 ]2507 }2508 ,2509 {2510 "pattern": "dcrawl",2511 "addition_date": "2017/09/22",2512 "instances": [2513 "dcrawl/1.0"2514 ]2515 }2516 ,2517 {2518 "pattern": "Snacktory",2519 "addition_date": "2017/09/23",2520 "url": "https://github.com/karussell/snacktory",2521 "instances": [2522 "Mozilla/5.0 (compatible; Snacktory; +https://github.com/karussell/snacktory)"2523 ]2524 }2525 ,2526 {2527 "pattern": "AndersPinkBot",2528 "addition_date": "2017/09/24",2529 "url": "http://anderspink.com/bot.html",2530 "instances": [2531 "Mozilla/5.0 (compatible; AndersPinkBot/1.0; +http://anderspink.com/bot.html)"2532 ]2533 }2534 ,2535 {2536 "pattern": "Fyrebot",2537 "addition_date": "2017/09/22",2538 "instances": [2539 "Fyrebot/1.0"2540 ]2541 }2542 ,2543 {2544 "pattern": "EveryoneSocialBot",2545 "addition_date": "2017/09/22",2546 "url": "http://everyonesocial.com",2547 "instances": [2548 "Mozilla/5.0 (compatible; EveryoneSocialBot/1.0; support@everyonesocial.com http://everyonesocial.com/)"2549 ]2550 }2551 ,2552 {2553 "pattern": "Mediatoolkitbot",2554 "addition_date": "2017/10/06",2555 "url": "http://mediatoolkit.com",2556 "instances": [2557 "Mediatoolkitbot (complaints@mediatoolkit.com)"2558 ]2559 }2560 ,2561 {2562 "pattern": "Luminator-robots",2563 "addition_date": "2017/09/22",2564 "instances": [2565 "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/30.0.1599.66 Safari/537.13 Luminator-robots/2.0"2566 ]2567 }2568 ,2569 {2570 "pattern": "ExtLinksBot",2571 "addition_date": "2017/11/02",2572 "url": "https://extlinks.com/Bot.html",2573 "instances": [2574 "Mozilla/5.0 (compatible; ExtLinksBot/1.5 +https://extlinks.com/Bot.html)"2575 ]2576 }2577 ,2578 {2579 "pattern": "SurveyBot",2580 "addition_date": "2017/11/02",2581 "instances": [2582 "Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.9.0.13) Gecko/2009073022 Firefox/3.5.2 (.NET CLR 3.5.30729) SurveyBot/2.3 (DomainTools)"2583 ]2584 }2585 ,2586 {2587 "pattern": "NING\\/",2588 "addition_date": "2017/11/02",2589 "instances": [2590 "NING/1.0"2591 ]2592 }2593 ,2594 {2595 "pattern": "okhttp",2596 "addition_date": "2017/11/02",2597 "instances": [2598 "okhttp/2.5.0",2599 "okhttp/2.7.5",2600 "okhttp/3.2.0",2601 "okhttp/3.5.0",2602 "okhttp/4.1.0"2603 ]2604 }2605 ,2606 {2607 "pattern": "Nuzzel",2608 "addition_date": "2017/11/02",2609 "instances": [2610 "Nuzzel"2611 ]2612 }2613 ,2614 {2615 "pattern": "omgili",2616 "addition_date": "2017/11/02",2617 "url": "http://omgili.com",2618 "instances": [2619 "omgili/0.5 +http://omgili.com"2620 ]2621 }2622 ,2623 {2624 "pattern": "PocketParser",2625 "addition_date": "2017/11/02",2626 "url": "https://getpocket.com/pocketparser_ua",2627 "instances": [2628 "PocketParser/2.0 (+https://getpocket.com/pocketparser_ua)"2629 ]2630 }2631 ,2632 {2633 "pattern": "YisouSpider",2634 "addition_date": "2017/11/02",2635 "instances": [2636 "YisouSpider",2637 "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 YisouSpider/5.0 Safari/537.36"2638 ]2639 }2640 ,2641 {2642 "pattern": "um-LN",2643 "addition_date": "2017/11/02",2644 "instances": [2645 "Mozilla/5.0 (compatible; um-LN/1.0; mailto: techinfo@ubermetrics-technologies.com)"2646 ]2647 }2648 ,2649 {2650 "pattern": "ToutiaoSpider",2651 "addition_date": "2017/11/02",2652 "url": "http://web.toutiao.com/media_cooperation/",2653 "instances": [2654 "Mozilla/5.0 (compatible; ToutiaoSpider/1.0; http://web.toutiao.com/media_cooperation/;)"2655 ]2656 }2657 ,2658 {2659 "pattern": "MuckRack",2660 "addition_date": "2017/11/02",2661 "url": "http://muckrack.com",2662 "instances": [2663 "Mozilla/5.0 (compatible; MuckRack/1.0; +http://muckrack.com)"2664 ]2665 }2666 ,2667 {2668 "pattern": "Jamie's Spider",2669 "addition_date": "2017/11/02",2670 "url": "http://jamiembrown.com/",2671 "instances": [2672 "Jamie's Spider (http://jamiembrown.com/)"2673 ]2674 }2675 ,2676 {2677 "pattern": "AHC\\/",2678 "addition_date": "2017/11/02",2679 "instances": [2680 "AHC/2.0"2681 ]2682 }2683 ,2684 {2685 "pattern": "NetcraftSurveyAgent",2686 "addition_date": "2017/11/02",2687 "instances": [2688 "Mozilla/5.0 (compatible; NetcraftSurveyAgent/1.0; +info@netcraft.com)"2689 ]2690 }2691 ,2692 {2693 "pattern": "Laserlikebot",2694 "addition_date": "2017/11/02",2695 "instances": [2696 "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Laserlikebot/0.1)"2697 ]2698 }2699 ,2700 {2701 "pattern": "^Apache-HttpClient",2702 "addition_date": "2017/11/02",2703 "instances": [2704 "Apache-HttpClient/4.2.3 (java 1.5)",2705 "Apache-HttpClient/4.2.5 (java 1.5)",2706 "Apache-HttpClient/4.3.1 (java 1.5)",2707 "Apache-HttpClient/4.3.3 (java 1.5)",2708 "Apache-HttpClient/4.3.5 (java 1.5)",2709 "Apache-HttpClient/4.4.1 (Java/1.8.0_65)",2710 "Apache-HttpClient/4.5.2 (Java/1.8.0_65)",2711 "Apache-HttpClient/4.5.2 (Java/1.8.0_151)",2712 "Apache-HttpClient/4.5.2 (Java/1.8.0_161)",2713 "Apache-HttpClient/4.5.2 (Java/1.8.0_181)",2714 "Apache-HttpClient/4.5.3 (Java/1.8.0_121)",2715 "Apache-HttpClient/4.5.3-SNAPSHOT (Java/1.8.0_152)",2716 "Apache-HttpClient/4.5.7 (Java/11.0.3)",2717 "Apache-HttpClient/4.5.10 (Java/1.8.0_201)"2718 ]2719 }2720 ,2721 {2722 "pattern": "AppEngine-Google",2723 "addition_date": "2017/11/02",2724 "instances": [2725 "AppEngine-Google; (+http://code.google.com/appengine; appid: example)",2726 "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36 AppEngine-Google; (+http://code.google.com/appengine; appid: s~feedly-nikon3)"2727 ]2728 }2729 ,2730 {2731 "pattern": "Jetty",2732 "addition_date": "2017/11/02",2733 "instances": [2734 "Jetty/9.3.z-SNAPSHOT"2735 ]2736 }2737 ,2738 {2739 "pattern": "Upflow",2740 "addition_date": "2017/11/02",2741 "instances": [2742 "Upflow/1.0"2743 ]2744 }2745 ,2746 {2747 "pattern": "Thinklab",2748 "addition_date": "2017/11/02",2749 "url": "thinklab.com",2750 "instances": [2751 "Thinklab (thinklab.com)"2752 ]2753 }2754 ,2755 {2756 "pattern": "Traackr.com",2757 "addition_date": "2017/11/02",2758 "url": "Traackr.com",2759 "instances": [2760 "Traackr.com"2761 ]2762 }2763 ,2764 {2765 "pattern": "Twurly",2766 "addition_date": "2017/11/02",2767 "url": "http://twurly.org",2768 "instances": [2769 "Ruby, Twurly v1.1 (http://twurly.org)"2770 ]2771 }2772 ,2773 {2774 "pattern": "Mastodon",2775 "addition_date": "2017/11/02",2776 "instances": [2777 "http.rb/2.2.2 (Mastodon/1.5.1; +https://example-masto-instance.org/)"2778 ]2779 }2780 ,2781 {2782 "pattern": "http_get",2783 "addition_date": "2017/11/02",2784 "instances": [2785 "http_get"2786 ]2787 }2788 ,2789 {2790 "pattern": "DnyzBot",2791 "addition_date": "2017/11/20",2792 "instances": [2793 "Mozilla/5.0 (compatible; DnyzBot/1.0)"2794 ]2795 }2796 ,2797 {2798 "pattern": "botify",2799 "addition_date": "2018/02/01",2800 "instances": [2801 "Mozilla/5.0 (compatible; botify; http://botify.com)"2802 ]2803 }2804 ,2805 {2806 "pattern": "007ac9 Crawler",2807 "addition_date": "2018/02/09",2808 "instances": [2809 "Mozilla/5.0 (compatible; 007ac9 Crawler; http://crawler.007ac9.net/)"2810 ]2811 }2812 ,2813 {2814 "pattern": "BehloolBot",2815 "addition_date": "2018/02/09",2816 "instances": [2817 "Mozilla/5.0 (compatible; BehloolBot/beta; +http://www.webeaver.com/bot)"2818 ]2819 }2820 ,2821 {2822 "pattern": "BrandVerity",2823 "addition_date": "2018/02/27",2824 "instances": [2825 "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:41.0) Gecko/20100101 Firefox/55.0 BrandVerity/1.0 (http://www.brandverity.com/why-is-brandverity-visiting-me)",2826 "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Mobile/11A465 Twitter for iPhone BrandVerity/1.0 (http://www.brandverity.com/why-is-brandverity-visiting-me)"2827 ],2828 "url": "http://www.brandverity.com/why-is-brandverity-visiting-me"2829 }2830 ,2831 {2832 "pattern": "check_http",2833 "addition_date": "2018/02/09",2834 "instances": [2835 "check_http/v2.2.1 (nagios-plugins 2.2.1)"2836 ]2837 }2838 ,2839 {2840 "pattern": "BDCbot",2841 "addition_date": "2018/02/09",2842 "instances": [2843 "Mozilla/5.0 (Windows NT 6.1; compatible; BDCbot/1.0; +http://bigweb.bigdatacorp.com.br/faq.aspx) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Safari/537.36",2844 "Mozilla/5.0 (Windows NT 10.0; Win64; x64; BDCbot/1.0; +http://bigweb.bigdatacorp.com.br/faq.aspx) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36"2845 ]2846 }2847 ,2848 {2849 "pattern": "ZumBot",2850 "addition_date": "2018/02/09",2851 "instances": [2852 "Mozilla/5.0 (compatible; ZumBot/1.0; http://help.zum.com/inquiry)"2853 ]2854 }2855 ,2856 {2857 "pattern": "EZID",2858 "addition_date": "2018/02/09",2859 "instances": [2860 "EZID (EZID link checker; https://ezid.cdlib.org/)"2861 ]2862 }2863 ,2864 {2865 "pattern": "ICC-Crawler",2866 "addition_date": "2018/02/28",2867 "instances": [2868 "ICC-Crawler/2.0 (Mozilla-compatible; ; http://ucri.nict.go.jp/en/icccrawler.html)"2869 ],2870 "url": "http://ucri.nict.go.jp/en/icccrawler.html"2871 }2872 ,2873 {2874 "pattern": "ArchiveBot",2875 "addition_date": "2018/02/28",2876 "instances": [2877 "ArchiveTeam ArchiveBot/20170106.02 (wpull 2.0.2)"2878 ],2879 "url": "https://github.com/ArchiveTeam/ArchiveBot"2880 }2881 ,2882 {2883 "pattern": "^LCC ",2884 "addition_date": "2018/02/28",2885 "instances": [2886 "LCC (+http://corpora.informatik.uni-leipzig.de/crawler_faq.html)"2887 ],2888 "url": "http://corpora.informatik.uni-leipzig.de/crawler_faq.html"2889 }2890 ,2891 {2892 "pattern": "filterdb.iss.net\\/crawler",2893 "addition_date": "2018/03/16",2894 "instances": [2895 "Mozilla/5.0 (compatible; oBot/2.3.1; +http://filterdb.iss.net/crawler/)"2896 ],2897 "url": "http://filterdb.iss.net/crawler/"2898 }2899 ,2900 {2901 "pattern": "BLP_bbot",2902 "addition_date": "2018/03/27",2903 "instances": [2904 "BLP_bbot/0.1"2905 ]2906 }2907 ,2908 {2909 "pattern": "BomboraBot",2910 "addition_date": "2018/03/27",2911 "instances": [2912 "Mozilla/5.0 (compatible; BomboraBot/1.0; +http://www.bombora.com/bot)"2913 ],2914 "url": "http://www.bombora.com/bot"2915 }2916 ,2917 {2918 "pattern": "Buck\\/",2919 "addition_date": "2018/03/27",2920 "instances": [2921 "Buck/2.2; (+https://app.hypefactors.com/media-monitoring/about.html)"2922 ],2923 "url": "https://app.hypefactors.com/media-monitoring/about.html"2924 }2925 ,2926 {2927 "pattern": "Companybook-Crawler",2928 "addition_date": "2018/03/27",2929 "instances": [2930 "Companybook-Crawler (+https://www.companybooknetworking.com/)"2931 ],2932 "url": "https://www.companybooknetworking.com/"2933 }2934 ,2935 {2936 "pattern": "Genieo",2937 "addition_date": "2018/03/27",2938 "instances": [2939 "Mozilla/5.0 (compatible; Genieo/1.0 http://www.genieo.com/webfilter.html)"2940 ],2941 "url": "http://www.genieo.com/webfilter.html"2942 }2943 ,2944 {2945 "pattern": "magpie-crawler",2946 "addition_date": "2018/03/27",2947 "instances": [2948 "magpie-crawler/1.1 (U; Linux amd64; en-GB; +http://www.brandwatch.net)"2949 ],2950 "url": "http://www.brandwatch.net"2951 }2952 ,2953 {2954 "pattern": "MeltwaterNews",2955 "addition_date": "2018/03/27",2956 "instances": [2957 "MeltwaterNews www.meltwater.com"2958 ],2959 "url": "http://www.meltwater.com"2960 }2961 ,2962 {2963 "pattern": "Moreover",2964 "addition_date": "2018/03/27",2965 "instances": [2966 "Mozilla/5.0 Moreover/5.1 (+http://www.moreover.com)"2967 ],2968 "url": "http://www.moreover.com"2969 }2970 ,2971 {2972 "pattern": "newspaper\\/",2973 "addition_date": "2018/03/27",2974 "instances": [2975 "newspaper/0.1.0.7",2976 "newspaper/0.2.5",2977 "newspaper/0.2.6",2978 "newspaper/0.2.8"2979 ]2980 }2981 ,2982 {2983 "pattern": "ScoutJet",2984 "addition_date": "2018/03/27",2985 "instances": [2986 "Mozilla/5.0 (compatible; ScoutJet; +http://www.scoutjet.com/)"2987 ],2988 "url": "http://www.scoutjet.com/"2989 }2990 ,2991 {2992 "pattern": "(^| )sentry\\/",2993 "addition_date": "2018/03/27",2994 "instances": [2995 "sentry/8.22.0 (https://sentry.io)"2996 ],2997 "url": "https://sentry.io"2998 }2999 ,3000 {3001 "pattern": "StorygizeBot",3002 "addition_date": "2018/03/27",3003 "instances": [3004 "Mozilla/5.0 (compatible; StorygizeBot; http://www.storygize.com)"3005 ],3006 "url": "http://www.storygize.com"3007 }3008 ,3009 {3010 "pattern": "UptimeRobot",3011 "addition_date": "2018/03/27",3012 "instances": [3013 "Mozilla/5.0+(compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)"3014 ],3015 "url": "http://www.uptimerobot.com/"3016 }3017 ,3018 {3019 "pattern": "OutclicksBot",3020 "addition_date": "2018/04/21",3021 "instances": [3022 "OutclicksBot/2 +https://www.outclicks.net/agent/VjzDygCuk4ubNmg40ZMbFqT0sIh7UfOKk8s8ZMiupUR",3023 "OutclicksBot/2 +https://www.outclicks.net/agent/gIYbZ38dfAuhZkrFVl7sJBFOUhOVct6J1SvxgmBZgCe",3024 "OutclicksBot/2 +https://www.outclicks.net/agent/PryJzTl8POCRHfvEUlRN5FKtZoWDQOBEvFJ2wh6KH5J",3025 "OutclicksBot/2 +https://www.outclicks.net/agent/p2i4sNUh7eylJF1S6SGgRs5mP40ExlYvsr9GBxVQG6h"3026 ],3027 "url": "https://www.outclicks.net"3028 }3029 ,3030 {3031 "pattern": "seoscanners",3032 "addition_date": "2018/05/27",3033 "instances": [3034 "Mozilla/5.0 (compatible; seoscanners.net/1; +spider@seoscanners.net)"3035 ],3036 "url": "http://www.seoscanners.net/"3037 }3038 ,3039 {3040 "pattern": "Hatena",3041 "addition_date": "2018/05/29",3042 "instances": [3043 "Hatena Antenna/0.3",3044 "Hatena::Russia::Crawler/0.01",3045 "Hatena-Favicon/2 (http://www.hatena.ne.jp/faq/)",3046 "Hatena::Scissors/0.01",3047 "HatenaBookmark/4.0 (Hatena::Bookmark; Analyzer)",3048 "Hatena::Fetcher/0.01 (master) Furl/3.13"3049 ]3050 }3051 ,3052 {3053 "pattern": "Google Web Preview",3054 "addition_date": "2018/05/31",3055 "instances": [3056 "Mozilla/5.0 (Linux; U; Android 2.3.4; generic) AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview) Version/4.0 Mobile Safari/537.36",3057 "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview) Chrome/27.0.1453 Safari/537.36"3058 ]3059 }3060 ,3061 {3062 "pattern": "MauiBot",3063 "addition_date": "2018/06/06",3064 "instances": [3065 "MauiBot (crawler.feedback+wc@gmail.com)"3066 ]3067 }3068 ,3069 {3070 "pattern": "AlphaBot",3071 "addition_date": "2018/05/27",3072 "instances": [3073 "Mozilla/5.0 (compatible; AlphaBot/3.2; +http://alphaseobot.com/bot.html)"3074 ],3075 "url": "http://alphaseobot.com/bot.html"3076 }3077 ,3078 {3079 "pattern": "SBL-BOT",3080 "addition_date": "2018/06/06",3081 "instances": [3082 "SBL-BOT (http://sbl.net)"3083 ],3084 "url": "http://sbl.net",3085 "description" : "Bot of SoftByte BlackWidow"3086 }3087 ,3088 {3089 "pattern": "IAS crawler",3090 "addition_date": "2018/06/06",3091 "instances": [3092 "IAS crawler (ias_crawler; http://integralads.com/site-indexing-policy/)"3093 ],3094 "url": "http://integralads.com/site-indexing-policy/",3095 "description" : "Bot of Integral Ad Science, Inc."3096 }3097 ,3098 {3099 "pattern": "adscanner",3100 "addition_date": "2018/06/24",3101 "instances": [3102 "Mozilla/5.0 (compatible; adscanner/)"3103 ]3104 }3105 ,3106 {3107 "pattern": "Netvibes",3108 "addition_date": "2018/06/24",3109 "instances": [3110 "Netvibes (crawler/bot; http://www.netvibes.com",3111 "Netvibes (crawler; http://www.netvibes.com)"3112 ],3113 "url": "http://www.netvibes.com"3114 }3115 ,3116 {3117 "pattern": "acapbot",3118 "addition_date": "2018/06/27",3119 "instances": [3120 "Mozilla/5.0 (compatible;acapbot/0.1;treat like Googlebot)",3121 "Mozilla/5.0 (compatible;acapbot/0.1.;treat like Googlebot)"3122 ]3123 }3124 ,3125 {3126 "pattern": "Baidu-YunGuanCe",3127 "addition_date": "2018/06/27",3128 "instances": [3129 "Baidu-YunGuanCe-Bot(ce.baidu.com)",3130 "Baidu-YunGuanCe-SLABot(ce.baidu.com)",3131 "Baidu-YunGuanCe-ScanBot(ce.baidu.com)",3132 "Baidu-YunGuanCe-PerfBot(ce.baidu.com)",3133 "Baidu-YunGuanCe-VSBot(ce.baidu.com)"3134 ],3135 "url": "https://ce.baidu.com/topic/topic20150908",3136 "description": "Baidu Cloud Watch"3137 }3138 ,3139 {3140 "pattern": "bitlybot",3141 "addition_date": "2018/06/27",3142 "instances": [3143 "bitlybot/3.0 (+http://bit.ly/)",3144 "bitlybot/2.0",3145 "bitlybot"3146 ],3147 "url": "http://bit.ly/"3148 }3149 ,3150 {3151 "pattern": "blogmuraBot",3152 "addition_date": "2018/06/27",3153 "instances": [3154 "blogmuraBot (+http://www.blogmura.com)"3155 ],3156 "url": "http://www.blogmura.com",3157 "description": "A blog ranking site which links to blogs on just about every theme possible."3158 }3159 ,3160 {3161 "pattern": "Bot.AraTurka.com",3162 "addition_date": "2018/06/27",3163 "instances": [3164 "Bot.AraTurka.com/0.0.1"3165 ],3166 "url": "http://www.araturka.com"3167 }3168 ,3169 {3170 "pattern": "bot-pge.chlooe.com",3171 "addition_date": "2018/06/27",3172 "instances": [3173 "bot-pge.chlooe.com/1.0.0 (+http://www.chlooe.com/)"3174 ]3175 }3176 ,3177 {3178 "pattern": "BoxcarBot",3179 "addition_date": "2018/06/27",3180 "instances": [3181 "Mozilla/5.0 (compatible; BoxcarBot/1.1; +awesome@boxcar.io)"3182 ],3183 "url": "https://boxcar.io/"3184 }3185 ,3186 {3187 "pattern": "BTWebClient",3188 "addition_date": "2018/06/27",3189 "instances": [3190 "BTWebClient/180B(9704)"3191 ],3192 "url": "http://www.utorrent.com/",3193 "description": "µTorrent BitTorrent Client"3194 }3195 ,3196 {3197 "pattern": "ContextAd Bot",3198 "addition_date": "2018/06/27",3199 "instances": [3200 "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0;.NET CLR 1.0.3705; ContextAd Bot 1.0)",3201 "ContextAd Bot 1.0"3202 ]3203 }3204 ,3205 {3206 "pattern": "Digincore bot",3207 "addition_date": "2018/06/27",3208 "instances": [3209 "Mozilla/5.0 (compatible; Digincore bot; https://www.digincore.com/crawler.html for rules and instructions.)"3210 ],3211 "url": "http://www.digincore.com/crawler.html"3212 }3213 ,3214 {3215 "pattern": "Disqus",3216 "addition_date": "2018/06/27",3217 "instances": [3218 "Disqus/1.0"3219 ],3220 "url": "https://disqus.com/",3221 "description": "validate and quality check pages."3222 }3223 ,3224 {3225 "pattern": "Feedly",3226 "addition_date": "2018/06/27",3227 "instances": [3228 "Feedly/1.0 (+http://www.feedly.com/fetcher.html; like FeedFetcher-Google)",3229 "FeedlyBot/1.0 (http://feedly.com)"3230 ],3231 "url": "https://www.feedly.com/fetcher.html",3232 "description": "Feedly Fetcher is how Feedly grabs RSS or Atom feeds when users choose to add them to their Feedly or any of the other applications built on top of the feedly cloud."3233 }3234 ,3235 {3236 "pattern": "Fetch\\/",3237 "addition_date": "2018/06/27",3238 "instances": [3239 "Fetch/2.0a (CMS Detection/Web/SEO analysis tool, see http://guess.scritch.org)"3240 ]3241 }3242 ,3243 {3244 "pattern": "Fever",3245 "addition_date": "2018/06/27",3246 "instances": [3247 "Fever/1.38 (Feed Parser; http://feedafever.com; Allow like Gecko)"3248 ],3249 "url": "http://feedafever.com"3250 }3251 ,3252 {3253 "pattern": "Flamingo_SearchEngine",3254 "addition_date": "2018/06/27",3255 "instances": [3256 "Flamingo_SearchEngine (+http://www.flamingosearch.com/bot)"3257 ]3258 }3259 ,3260 {3261 "pattern": "FlipboardProxy",3262 "addition_date": "2018/06/27",3263 "instances": [3264 "Mozilla/5.0 (compatible; FlipboardProxy/1.1; +http://flipboard.com/browserproxy)",3265 "Mozilla/5.0 (compatible; FlipboardProxy/1.2; +http://flipboard.com/browserproxy)",3266 "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6 (FlipboardProxy/1.1; +http://flipboard.com/browserproxy)",3267 "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:28.0) Gecko/20100101 Firefox/28.0 (FlipboardProxy/1.1; +http://flipboard.com/browserproxy)",3268 "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:49.0) Gecko/20100101 Firefox/49.0 (FlipboardProxy/1.2; +http://flipboard.com/browserproxy)"3269 ],3270 "url": "https://about.flipboard.com/browserproxy/",3271 "description": "a proxy service to fetch, validate, and prepare certain elements of websites for presentation through the Flipboard Application"3272 }3273 ,3274 {3275 "pattern": "g2reader-bot",3276 "addition_date": "2018/06/27",3277 "instances": [3278 "g2reader-bot/1.0 (+http://www.g2reader.com/)"3279 ],3280 "url": "http://www.g2reader.com/"3281 }3282 ,3283 {3284 "pattern": "G2 Web Services",3285 "addition_date": "2019/03/01",3286 "instances": [3287 "G2 Web Services/1.0 (built with StormCrawler Archetype 1.8; https://www.g2webservices.com/; developers@g2llc.com)"3288 ],3289 "url": "https://www.g2webservices.com/"3290 }3291 ,3292 {3293 "pattern": "imrbot",3294 "addition_date": "2018/06/27",3295 "instances": [3296 "Mozilla/5.0 (compatible; imrbot/1.10.8 +http://www.mignify.com)"3297 ],3298 "url": "http://www.mignify.com"3299 }3300 ,3301 {3302 "pattern": "K7MLWCBot",3303 "addition_date": "2018/06/27",3304 "instances": [3305 "K7MLWCBot/1.0 (+http://www.k7computing.com)"3306 ],3307 "url": "http://www.k7computing.com",3308 "description": "Virus scanner"3309 }3310 ,3311 {3312 "pattern": "Kemvibot",3313 "addition_date": "2018/06/27",3314 "instances": [3315 "Kemvibot/1.0 (http://kemvi.com, marco@kemvi.com)"3316 ],3317 "url": "http://kemvi.com"3318 }3319 ,3320 {3321 "pattern": "Landau-Media-Spider",3322 "addition_date": "2018/06/27",3323 "instances": [3324 "Landau-Media-Spider/1.0(http://bots.landaumedia.de/bot.html)"3325 ],3326 "url": "http://bots.landaumedia.de/bot.html"3327 }3328 ,3329 {3330 "pattern": "linkapediabot",3331 "addition_date": "2018/06/27",3332 "instances": [3333 "linkapediabot (+http://www.linkapedia.com)"3334 ],3335 "url": "http://www.linkapedia.com"3336 }3337 ,3338 {3339 "pattern": "vkShare",3340 "addition_date": "2018/07/02",3341 "instances": [3342 "Mozilla/5.0 (compatible; vkShare; +http://vk.com/dev/Share)"3343 ],3344 "url": "http://vk.com/dev/Share"3345 }3346 ,3347 {3348 "pattern": "Siteimprove.com",3349 "addition_date": "2018/06/22",3350 "instances": [3351 "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) LinkCheck by Siteimprove.com",3352 "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.0) Match by Siteimprove.com",3353 "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) SiteCheck-sitecrawl by Siteimprove.com",3354 "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.0) LinkCheck by Siteimprove.com"3355 ]3356 }3357 ,3358 {3359 "pattern": "BLEXBot\\/",3360 "addition_date": "2018/07/07",3361 "instances": [3362 "Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)"3363 ],3364 "url": "http://webmeup-crawler.com"3365 }3366 ,3367 {3368 "pattern": "DareBoost",3369 "addition_date": "2018/07/07",3370 "instances": [3371 "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.75 Safari/537.36 DareBoost"3372 ],3373 "url": "https://www.dareboost.com/",3374 "description": "Bot to test, Analyze and Optimize website"3375 }3376 ,3377 {3378 "pattern": "ZuperlistBot\\/",3379 "addition_date": "2018/07/07",3380 "instances": [3381 "Mozilla/5.0 (compatible; ZuperlistBot/1.0)"3382 ]3383 }3384 ,3385 {3386 "pattern": "Miniflux\\/",3387 "addition_date": "2018/07/07",3388 "instances": [3389 "Mozilla/5.0 (compatible; Miniflux/2.0.x-dev; +https://miniflux.net)",3390 "Mozilla/5.0 (compatible; Miniflux/2.0.3; +https://miniflux.net)",3391 "Mozilla/5.0 (compatible; Miniflux/2.0.7; +https://miniflux.net)",3392 "Mozilla/5.0 (compatible; Miniflux/2.0.10; +https://miniflux.net)",3393 "Mozilla/5.0 (compatibl$; Miniflux/2.0.x-dev; +https://miniflux.app)",3394 "Mozilla/5.0 (compatible; Miniflux/2.0.11; +https://miniflux.app)",3395 "Mozilla/5.0 (compatible; Miniflux/2.0.12; +https://miniflux.app)",3396 "Mozilla/5.0 (compatible; Miniflux/ae1dc1a; +https://miniflux.app)",3397 "Mozilla/5.0 (compatible; Miniflux/3b6e44c; +https://miniflux.app)"3398 ],3399 "url": "https://miniflux.net",3400 "description": "Miniflux is a minimalist and opinionated feed reader."3401 }3402 ,3403 {3404 "pattern": "Feedspot",3405 "addition_date": "2018/07/07",3406 "instances": [3407 "Mozilla/5.0 (compatible; Feedspotbot/1.0; +http://www.feedspot.com/fs/bot)",3408 "Mozilla/5.0 (compatible; Feedspot/1.0 (+https://www.feedspot.com/fs/fetcher; like FeedFetcher-Google)"3409 ],3410 "url": "http://www.feedspot.com/fs/bot"3411 }3412 ,3413 {3414 "pattern": "Diffbot\\/",3415 "addition_date": "2018/07/07",3416 "instances": [3417 "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.2) Gecko/20090729 Firefox/3.5.2 (.NET CLR 3.5.30729; Diffbot/0.1; +http://www.diffbot.com)"3418 ],3419 "url": "http://www.diffbot.com"3420 }3421 ,3422 {3423 "pattern": "SEOkicks",3424 "addition_date": "2018/08/22",3425 "instances": [3426 "Mozilla/5.0 (compatible; SEOkicks; +https://www.seokicks.de/robot.html)"3427 ],3428 "url": "https://www.seokicks.de/robot.html"3429 }3430 ,3431 {3432 "pattern": "tracemyfile",3433 "addition_date": "2018/08/23",3434 "instances": [3435 "Mozilla/5.0 (compatible; tracemyfile/1.0; +bot@tracemyfile.com)"3436 ]3437 }3438 ,3439 {3440 "pattern": "Nimbostratus-Bot",3441 "addition_date": "2018/08/29",3442 "instances": [3443 "Mozilla/5.0 (compatible; Nimbostratus-Bot/v1.3.2; http://cloudsystemnetworks.com)"3444 ]3445 }3446 ,3447 {3448 "pattern": "zgrab",3449 "addition_date": "2018/08/30",3450 "instances": [3451 "Mozilla/5.0 zgrab/0.x"3452 ],3453 "url": "https://zmap.io/"3454 }3455 ,3456 {3457 "pattern": "PR-CY.RU",3458 "addition_date": "2018/08/30",3459 "instances": [3460 "Mozilla/5.0 (compatible; PR-CY.RU; + https://a.pr-cy.ru)"3461 ],3462 "url": "https://a.pr-cy.ru/"3463 }3464 ,3465 {3466 "pattern": "AdsTxtCrawler",3467 "addition_date": "2018/08/30",3468 "instances": [3469 "AdsTxtCrawler/1.0"3470 ]3471 },3472 {3473 "pattern": "Datafeedwatch",3474 "addition_date": "2018/09/05",3475 "instances": [3476 "Datafeedwatch/2.1.x"3477 ],3478 "url": "https://www.datafeedwatch.com/"3479 }3480 ,3481 {3482 "pattern": "Zabbix",3483 "addition_date": "2018/09/05",3484 "instances": [3485 "Zabbix"3486 ],3487 "url": "https://www.zabbix.com/documentation/3.4/manual/web_monitoring"3488 }3489 ,3490 {3491 "pattern": "TangibleeBot",3492 "addition_date": "2018/09/05",3493 "instances": [3494 "TangibleeBot/1.0.0.0 (http://tangiblee.com/bot)"3495 ],3496 "url": "http://tangiblee.com/bot"3497 }3498 ,3499 {3500 "pattern": "google-xrawler",3501 "addition_date": "2018/09/05",3502 "instances": [3503 "google-xrawler"3504 ],3505 "url": "https://webmasters.stackexchange.com/questions/105560/what-is-the-google-xrawler-user-agent-used-for"3506 }3507 ,3508 {3509 "pattern": "axios",3510 "addition_date": "2018/09/06",3511 "instances": [3512 "axios/0.18.0",3513 "axios/0.19.0"3514 ],3515 "url": "https://github.com/axios/axios"3516 }3517 ,3518 {3519 "pattern": "Amazon CloudFront",3520 "addition_date": "2018/09/07",3521 "instances": [3522 "Amazon CloudFront"3523 ],3524 "url": "https://aws.amazon.com/cloudfront/"3525 }3526 ,3527 {3528 "pattern": "Pulsepoint",3529 "addition_date": "2018/09/24",3530 "instances": [3531 "Pulsepoint XT3 web scraper"3532 ]3533 }3534 ,3535 {3536 "pattern": "CloudFlare-AlwaysOnline",3537 "addition_date": "2018/09/27",3538 "instances": [3539 "Mozilla/5.0 (compatible; CloudFlare-AlwaysOnline/1.0; +http://www.cloudflare.com/always-online) AppleWebKit/534.34",3540 "Mozilla/5.0 (compatible; CloudFlare-AlwaysOnline/1.0; +https://www.cloudflare.com/always-online) AppleWebKit/534.34"3541 ],3542 "url" : "https://www.cloudflare.com/always-online/"3543 }3544 ,3545 {3546 "pattern": "Google-Structured-Data-Testing-Tool",3547 "addition_date": "2018/10/02",3548 "instances": [3549 "Mozilla/5.0 (compatible; Google-Structured-Data-Testing-Tool +https://search.google.com/structured-data/testing-tool)",3550 "Mozilla/5.0 (compatible; Google-Structured-Data-Testing-Tool +http://developers.google.com/structured-data/testing-tool/)"3551 ],3552 "url": "https://search.google.com/structured-data/testing-tool"3553 }3554 ,3555 {3556 "pattern": "WordupInfoSearch",3557 "addition_date": "2018/10/07",3558 "instances": [3559 "WordupInfoSearch/1.0"3560 ]3561 }3562 ,3563 {3564 "pattern": "WebDataStats",3565 "addition_date": "2018/10/08",3566 "instances": [3567 "Mozilla/5.0 (compatible; WebDataStats/1.0 ; +https://webdatastats.com/policy.html)"3568 ],3569 "url": "https://webdatastats.com/"3570 }3571 ,3572 {3573 "pattern": "HttpUrlConnection",3574 "addition_date": "2018/10/08",3575 "instances": [3576 "Jersey/2.25.1 (HttpUrlConnection 1.8.0_141)"3577 ]3578 }3579 ,3580 {3581 "pattern": "Seekport Crawler",3582 "addition_date": "2018/10/08",3583 "instances": [3584 "Mozilla/5.0 (compatible; Seekport Crawler; http://seekport.com/)"3585 ],3586 "url": "http://seekport.com/"3587 }3588 ,3589 {3590 "pattern": "ZoomBot",3591 "addition_date": "2018/10/10",3592 "instances": [3593 "ZoomBot (Linkbot 1.0 http://suite.seozoom.it/bot.html)"3594 ],3595 "url": "http://suite.seozoom.it/bot.html"3596 }3597 ,3598 {3599 "pattern": "VelenPublicWebCrawler",3600 "addition_date": "2018/10/09",3601 "instances": [3602 "VelenPublicWebCrawler (velen.io)"3603 ]3604 }3605 ,3606 {3607 "pattern": "MoodleBot",3608 "addition_date": "2018/10/10",3609 "instances": [3610 "MoodleBot/1.0"3611 ]3612 }3613 ,3614 {3615 "pattern": "jpg-newsbot",3616 "addition_date": "2018/10/10",3617 "instances": [3618 "jpg-newsbot/2.0; (+https://vipnytt.no/bots/)"3619 ],3620 "url": "https://vipnytt.no/bots/"3621 }3622 ,3623 {3624 "pattern": "outbrain",3625 "addition_date": "2018/10/14",3626 "instances": [3627 "Mozilla/5.0 (Java) outbrain"3628 ],3629 "url": "https://www.outbrain.com/help/advertisers/invalid-url/"3630 }3631 ,3632 {3633 "pattern": "W3C_Validator",3634 "addition_date": "2018/10/14",3635 "instances": [3636 "W3C_Validator/1.3"3637 ],3638 "url": "https://validator.w3.org/services"3639 }3640 ,3641 {3642 "pattern": "Validator\\.nu",3643 "addition_date": "2018/10/14",3644 "instances": [3645 "Validator.nu/LV"3646 ],3647 "url": "https://validator.w3.org/services"3648 }3649 ,3650 {3651 "pattern": "W3C-checklink",3652 "addition_date": "2018/10/14",3653 "depends_on": ["libwww-perl"],3654 "instances": [3655 "W3C-checklink/2.90 libwww-perl/5.64",3656 "W3C-checklink/3.6.2.3 libwww-perl/5.64",3657 "W3C-checklink/4.2 [4.20] libwww-perl/5.803",3658 "W3C-checklink/4.2.1 [4.21] libwww-perl/5.803",3659 "W3C-checklink/4.3 [4.42] libwww-perl/5.805",3660 "W3C-checklink/4.3 [4.42] libwww-perl/5.808",3661 "W3C-checklink/4.3 [4.42] libwww-perl/5.820",3662 "W3C-checklink/4.5 [4.154] libwww-perl/5.823",3663 "W3C-checklink/4.5 [4.160] libwww-perl/5.823"3664 ],3665 "url": "https://validator.w3.org/services"3666 }3667 ,3668 {3669 "pattern": "W3C-mobileOK",3670 "addition_date": "2018/10/14",3671 "instances": [3672 "W3C-mobileOK/DDC-1.0"3673 ],3674 "url": "https://validator.w3.org/services"3675 }3676 ,3677 {3678 "pattern": "W3C_I18n-Checker",3679 "addition_date": "2018/10/14",3680 "instances": [3681 "W3C_I18n-Checker/1.0"3682 ],3683 "url": "https://validator.w3.org/services"3684 }3685 ,3686 {3687 "pattern": "FeedValidator",3688 "addition_date": "2018/10/14",3689 "instances": [3690 "FeedValidator/1.3"3691 ],3692 "url": "https://validator.w3.org/services"3693 }3694 ,3695 {3696 "pattern": "W3C_CSS_Validator",3697 "addition_date": "2018/10/14",3698 "instances": [3699 "Jigsaw/2.3.0 W3C_CSS_Validator_JFouffa/2.0"3700 ],3701 "url": "https://validator.w3.org/services"3702 }3703 ,3704 {3705 "pattern": "W3C_Unicorn",3706 "addition_date": "2018/10/14",3707 "instances": [3708 "W3C_Unicorn/1.0"3709 ],3710 "url": "https://validator.w3.org/services"3711 }3712 ,3713 {3714 "pattern": "Google-PhysicalWeb",3715 "addition_date": "2018/10/21",3716 "instances": [3717 "Mozilla/5.0 (Google-PhysicalWeb)"3718 ]3719 }3720 ,3721 {3722 "pattern": "Blackboard",3723 "addition_date": "2018/10/28",3724 "instances": [3725 "Blackboard Safeassign"3726 ],3727 "url": "https://help.blackboard.com/Learn/Administrator/Hosting/Tools_Management/SafeAssign"3728 },3729 {3730 "pattern": "ICBot\\/",3731 "addition_date": "2018/10/23",3732 "instances": [3733 "Mozilla/5.0 (compatible; ICBot/0.1; +https://ideasandcode.xyz"3734 ],3735 "url": "https://ideasandcode.xyz"3736 },3737 {3738 "pattern": "BazQux",3739 "addition_date": "2018/10/23",3740 "instances": [3741 "Mozilla/5.0 (compatible; BazQux/2.4; +https://bazqux.com/fetcher; 1 subscribers)"3742 ],3743 "url": "https://bazqux.com/fetcher"3744 },3745 {3746 "pattern": "Twingly",3747 "addition_date": "2018/10/23",3748 "instances": [3749 "Mozilla/5.0 (compatible; Twingly Recon; twingly.com)"3750 ],3751 "url": "https://twingly.com"3752 },3753 {3754 "pattern": "Rivva",3755 "addition_date": "2018/10/23",3756 "instances": [3757 "Mozilla/5.0 (compatible; Rivva; http://rivva.de)"3758 ],3759 "url": "http://rivva.de"3760 },3761 {3762 "pattern": "Experibot",3763 "addition_date": "2018/11/03",3764 "instances": [3765 "Experibot-v2 http://goo.gl/ZAr8wX",3766 "Experibot-v3 http://goo.gl/ZAr8wX"3767 ],3768 "url": "https://amirkr.wixsite.com/experibot"3769 },3770 {3771 "pattern": "awesomecrawler",3772 "addition_date": "2018/11/24",3773 "instances": [3774 "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.22 (KHTML, like Gecko) Chrome/25.0.1364.5 Safari/537.22 +awesomecrawler"3775 ]3776 },3777 {3778 "pattern": "Dataprovider.com",3779 "addition_date": "2018/11/24",3780 "instances": [3781 "Mozilla/5.0 (compatible; Dataprovider.com)"3782 ],3783 "url": "https://www.dataprovider.com/"3784 },3785 {3786 "pattern": "GroupHigh\\/",3787 "addition_date": "2018/11/24",3788 "instances": [3789 "Mozilla/5.0 (compatible; GroupHigh/1.0; +http://www.grouphigh.com/"3790 ],3791 "url": "http://www.grouphigh.com/"3792 },3793 {3794 "pattern": "theoldreader.com",3795 "addition_date": "2018/12/02",3796 "instances": [3797 "Mozilla/5.0 (compatible; theoldreader.com)"3798 ],3799 "url": "https://www.theoldreader.com/"3800 }3801 ,3802 {3803 "pattern": "AnyEvent",3804 "addition_date": "2018/12/07",3805 "instances": [3806 "Mozilla/5.0 (compatible; U; AnyEvent-HTTP/2.24; +http://software.schmorp.de/pkg/AnyEvent)"3807 ],3808 "url": "http://software.schmorp.de/pkg/AnyEvent.html"3809 }3810 ,3811 {3812 "pattern": "Uptimebot\\.org",3813 "addition_date": "2019/01/17",3814 "instances": [3815 "Uptimebot.org - Free website monitoring"3816 ],3817 "url": "http://uptimebot.org/"3818 }3819 ,3820 {3821 "pattern": "Nmap Scripting Engine",3822 "addition_date": "2019/02/04",3823 "instances": [3824 "Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)"3825 ],3826 "url": "https://nmap.org/book/nse.html"3827 }3828 ,3829 {3830 "pattern": "2ip.ru",3831 "addition_date": "2019/02/12",3832 "instances": [3833 "2ip.ru CMS Detector (https://2ip.ru/cms/)"3834 ],3835 "url": "https://2ip.ru/cms/"3836 },3837 {3838 "pattern": "Clickagy",3839 "addition_date": "2019/02/19",3840 "instances": [3841 "Clickagy Intelligence Bot v2"3842 ],3843 "url": "https://www.clickagy.com"3844 },3845 {3846 "pattern": "Caliperbot",3847 "addition_date": "2019/03/02",3848 "instances": [3849 "Caliperbot/1.0 (+http://www.conductor.com/caliperbot)"3850 ],3851 "url": "http://www.conductor.com/caliperbot"3852 },3853 {3854 "pattern": "MBCrawler",3855 "addition_date": "2019/03/02",3856 "instances": [3857 "MBCrawler/1.0 (https://monitorbacklinks.com)"3858 ],3859 "url": "https://monitorbacklinks.com"3860 },3861 {3862 "pattern": "online-webceo-bot",3863 "addition_date": "2019/03/02",3864 "instances": [3865 "Mozilla/5.0 (compatible; online-webceo-bot/1.0; +http://online.webceo.com)"3866 ],3867 "url": "http://online.webceo.com"3868 },3869 {3870 "pattern": "B2B Bot",3871 "addition_date": "2019/03/02",3872 "instances": [3873 "B2B Bot"3874 ]3875 },3876 {3877 "pattern": "AddSearchBot",3878 "addition_date": "2019/03/02",3879 "instances": [3880 "Mozilla/5.0 (compatible; AddSearchBot/0.9; +http://www.addsearch.com/bot; info@addsearch.com)"3881 ],3882 "url": "http://www.addsearch.com/bot"3883 },3884 {3885 "pattern": "Google Favicon",3886 "addition_date": "2019/03/14",3887 "instances": [3888 "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36 Google Favicon"3889 ]3890 },3891 {3892 "pattern": "HubSpot",3893 "addition_date": "2019/04/15",3894 "instances": [3895 "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36 HubSpot Webcrawler - web-crawlers@hubspot.com",3896 "Mozilla/5.0 (X11; Linux x86_64; HubSpot Single Page link check; web-crawlers+links@hubspot.com) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36",3897 "Mozilla/5.0 (compatible; HubSpot Crawler; web-crawlers@hubspot.com)",3898 "HubSpot Connect 2.0 (http://dev.hubspot.com/) - BizOpsCompanies-Tq2-BizCoDomainValidationAudit"3899 ]3900 },3901 {3902 "pattern": "Chrome-Lighthouse",3903 "addition_date": "2019/03/15",3904 "instances": [3905 "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5 Build/MRA58N) AppleWebKit/537.36(KHTML, like Gecko) Chrome/69.0.3464.0 Mobile Safari/537.36 Chrome-Lighthouse",3906 "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36(KHTML, like Gecko) Chrome/69.0.3464.0 Safari/537.36 Chrome-Lighthouse",3907 "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3694.0 Safari/537.36 Chrome-Lighthouse",3908 "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3694.0 Mobile Safari/537.36 Chrome-Lighthouse"3909 ],3910 "url": "https://developers.google.com/speed/pagespeed/insights"3911 },3912 {3913 "pattern": "HeadlessChrome",3914 "url": "https://developers.google.com/web/updates/2017/04/headless-chrome",3915 "addition_date": "2019/06/17",3916 "instances": [3917 "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/74.0.3729.169 Safari/537.36",3918 "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/69.0.3494.0 Safari/537.36",3919 "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/76.0.3803.0 Safari/537.36"3920 ]3921 },3922 {3923 "pattern": "CheckMarkNetwork\\/",3924 "addition_date": "2019/06/30",3925 "instances": [3926 "CheckMarkNetwork/1.0 (+http://www.checkmarknetwork.com/spider.html)"3927 ],3928 "url": "https://www.checkmarknetwork.com/"3929 },3930 {3931 "pattern": "www\\.uptime\\.com",3932 "addition_date": "2019/07/21",3933 "instances": [3934 "Mozilla/5.0 (compatible; Uptimebot/1.0; +http://www.uptime.com/uptimebot)"3935 ],3936 "url": "http://www.uptime.com/uptimebot"3937 }3938 ,3939 {3940 "pattern": "Streamline3Bot\\/",3941 "addition_date": "2019/07/21",3942 "instances": [3943 "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1) Streamline3Bot/1.0",3944 "Mozilla/5.0 (Windows NT 6.1; Win64; x64; +https://www.ubtsupport.com/legal/Streamline3Bot.php) Streamline3Bot/1.0"3945 ],3946 "url": "https://www.ubtsupport.com/legal/Streamline3Bot.php"3947 }3948 ,3949 {3950 "pattern": "serpstatbot\\/",3951 "addition_date": "2019/07/25",3952 "instances": [3953 "serpstatbot/1.0 (advanced backlink tracking bot; http://serpstatbot.com/; abuse@serpstatbot.com)",3954 "serpstatbot/1.0 (advanced backlink tracking bot; curl/7.58.0; http://serpstatbot.com/; abuse@serpstatbot.com)"3955 ],3956 "url": "http://serpstatbot.com"3957 }3958 ,3959 {3960 "pattern": "MixnodeCache\\/",3961 "addition_date": "2019/08/04",3962 "instances": [3963 "MixnodeCache/1.8(+https://cache.mixnode.com/)"3964 ],3965 "url": "https://cache.mixnode.com/"3966 }3967 ,3968 {3969 "pattern": "^curl",3970 "addition_date": "2019/08/15",3971 "instances": [3972 "curl",3973 "curl/7.29.0",3974 "curl/7.47.0",3975 "curl/7.54.0",3976 "curl/7.55.1",3977 "curl/7.64.0",3978 "curl/7.64.1",3979 "curl/7.65.3"3980 ],3981 "url": "https://curl.haxx.se/"3982 }3983 ,3984 {3985 "pattern": "SimpleScraper",3986 "addition_date": "2019/08/16",3987 "instances": [3988 "Mozilla/5.0 (compatible; SimpleScraper)"3989 ],3990 "url": "https://github.com/ramonkcom/simple-scraper/"3991 }3992 ,3993 {3994 "pattern": "RSSingBot",3995 "addition_date": "2019/09/15",3996 "instances": [3997 "RSSingBot (http://www.rssing.com)"3998 ],3999 "url": "http://www.rssing.com"4000 }4001 ,4002 {4003 "pattern": "Jooblebot",4004 "addition_date": "2019/09/25",4005 "instances": [4006 "Mozilla/5.0 (compatible; Jooblebot/2.0; Windows NT 6.1; WOW64; +http://jooble.org/jooble-bot) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36"4007 ],4008 "url": "http://jooble.org/jooble-bot"4009 }4010 ,4011 {4012 "pattern": "fedoraplanet",4013 "addition_date": "2019/09/28",4014 "instances": [4015 "venus/fedoraplanet"4016 ],4017 "url": "http://fedoraplanet.org/"4018 }4019 ,4020 {4021 "pattern": "Friendica",4022 "addition_date": "2019/09/28",4023 "instances": [4024 "Friendica 'The Tazmans Flax-lily' 2019.01-1293; https://hoyer.xyz"4025 ],4026 "url": "https://hoyer.xyz"4027 }4028 ,4029 {4030 "pattern": "NextCloud",4031 "addition_date": "2019/09/30",4032 "instances": [4033 "NextCloud-News/1.0"4034 ],4035 "url": "https://nextcloud.com/"4036 }4037 ,4038 {4039 "pattern": "Tiny Tiny RSS",4040 "addition_date": "2019/10/04",4041 "instances": [4042 "Tiny Tiny RSS/1.15.3 (http://tt-rss.org/)",4043 "Tiny Tiny RSS/17.12 (a2d1fa5) (http://tt-rss.org/)",4044 "Tiny Tiny RSS/19.2 (b68db2d) (http://tt-rss.org/)",4045 "Tiny Tiny RSS/19.8 (http://tt-rss.org/)"4046 ],4047 "url": "http://tt-rss.org/"4048 }4049 ,4050 {4051 "pattern": "RegionStuttgartBot",4052 "addition_date": "2019/10/17",4053 "instances": [4054 "Mozilla/5.0 (compatible; RegionStuttgartBot/1.0; +http://it.region-stuttgart.de/competenzatlas/unternehmen-suchen/)"4055 ],4056 "url": "http://it.region-stuttgart.de/competenzatlas/unternehmen-suchen/"4057 }4058 ,4059 {4060 "pattern": "Bytespider",4061 "addition_date": "2019/11/11",4062 "instances": [4063 "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.3754.1902 Mobile Safari/537.36; Bytespider",4064 "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.4454.1745 Mobile Safari/537.36; Bytespider",4065 "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.7597.1164 Mobile Safari/537.36; Bytespider;bytespider@bytedance.com",4066 "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2988.1545 Mobile Safari/537.36; Bytespider",4067 "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.4141.1682 Mobile Safari/537.36; Bytespider",4068 "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.3478.1649 Mobile Safari/537.36; Bytespider",4069 "Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.5267.1259 Mobile Safari/537.36; Bytespider",4070 "Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.7990.1979 Mobile Safari/537.36; Bytespider",4071 "Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.2268.1523 Mobile Safari/537.36; Bytespider",4072 "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2576.1836 Mobile Safari/537.36; Bytespider",4073 "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.9681.1227 Mobile Safari/537.36; Bytespider",4074 "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.6023.1635 Mobile Safari/537.36; Bytespider",4075 "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.4944.1981 Mobile Safari/537.36; Bytespider",4076 "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.3613.1739 Mobile Safari/537.36; Bytespider",4077 "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.4022.1033 Mobile Safari/537.36; Bytespider",4078 "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.3248.1547 Mobile Safari/537.36; Bytespider",4079 "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.5527.1507 Mobile Safari/537.36; Bytespider",4080 "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.5216.1326 Mobile Safari/537.36; Bytespider",4081 "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.9038.1080 Mobile Safari/537.36; Bytespider"4082 ],4083 "url": "https://stackoverflow.com/questions/57908900/what-is-the-bytespider-user-agent"4084 }4085 ,4086 {4087 "pattern": "Datanyze",4088 "addition_date": "2019/11/17",4089 "instances": [4090 "Mozilla/5.0 (X11; Datanyze; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36"4091 ],4092 "url": "https://www.datanyze.com/dnyzbot/"4093 }4094 ,4095 {4096 "pattern": "Google-Site-Verification",4097 "addition_date": "2019/12/11",4098 "instances": [4099 "Mozilla/5.0 (compatible; Google-Site-Verification/1.0)"4100 ],4101 "url": "https://support.google.com/webmasters/answer/9008080"4102 }4103 ,4104 {4105 "pattern": "TrendsmapResolver",4106 "addition_date": "2020/02/24",4107 "instances": [4108 "Mozilla/5.0 (compatible; TrendsmapResolver/0.1)"4109 ],4110 "url": "https://www.trendsmap.com/"4111 }4112 ,4113 {4114 "pattern": "tweetedtimes",4115 "addition_date": "2020/02/24",4116 "instances": [4117 "Mozilla/5.0 (compatible; +http://tweetedtimes.com)"4118 ],4119 "url": "https://tweetedtimes.com/"4120 }4121 ]4122 `4123 var patterns []CrawlerPattern4124 err := json.Unmarshal([]byte(crawlers), &patterns)4125 if err != nil {4126 panic(err)4127 }4128 return patterns4129}...

tmpl_new_proj_adm_conf_secure.go

Source:tmpl_new_proj_adm_conf_secure.go

1// Copyright 2014 The goyy Authors. All rights reserved.2// Use of this source code is governed by a MIT-style3// license that can be found in the LICENSE file.45package main67var tmplNewProjAdmConfSecure = `<?xml version="1.0" encoding="UTF-8" ?>8<!DOCTYPE env PUBLIC "-//GOYY//DTD ENV 1.0//EN" "http://goyy.org/dtd/env-1.0.dtd">9<configuration>10 <environments default="development">11 <environment id="development">12 <secure name="<%.NewProjName%>">13 <enable>true</enable>14 <login-url>/login.html</login-url>15 <forbid-url>/err/403.html</forbid-url>16 <success-url>/home.html</success-url>17 <filters>18 <intercept-url pattern="/**.(css|js|map)" access="anon"/>19 <intercept-url pattern="/**.(jpg|gif|png|bmp|ico)" access="anon"/>20 <intercept-url pattern="/login.html" access="anon"/>21 <intercept-url pattern="/login" access="anon"/>22 <intercept-url pattern="/signin" access="anon"/>23 <intercept-url pattern="/logout" access="anon"/>24 <intercept-url pattern="/captcha/build" access="anon"/>25 <intercept-url pattern="/err/**" access="anon"/>26 <intercept-url pattern="/**" access="authc"/>27 </filters>28 </secure>29 </environment>30 <environment id="test">31 <secure name="<%.NewProjName%>">32 <enable>true</enable>33 <login-url>/login.html</login-url>34 <forbid-url>/err/403.html</forbid-url>35 <success-url>/home.html</success-url>36 <filters>37 <intercept-url pattern="/**.(css|js|map)" access="anon"/>38 <intercept-url pattern="/**.(jpg|gif|png|bmp|ico)" access="anon"/>39 <intercept-url pattern="/login.html" access="anon"/>40 <intercept-url pattern="/login" access="anon"/>41 <intercept-url pattern="/signin" access="anon"/>42 <intercept-url pattern="/logout" access="anon"/>43 <intercept-url pattern="/captcha/build" access="anon"/>44 <intercept-url pattern="/err/**" access="anon"/>45 <intercept-url pattern="/**" access="authc"/>46 </filters>47 </secure>48 </environment>49 <environment id="production">50 <secure name="<%.NewProjName%>">51 <enable>true</enable>52 <login-url>/login.html</login-url>53 <forbid-url>/err/403.html</forbid-url>54 <success-url>/home.html</success-url>55 <filters>56 <intercept-url pattern="/**.(css|js|map)" access="anon"/>57 <intercept-url pattern="/**.(jpg|gif|png|bmp|ico)" access="anon"/>58 <intercept-url pattern="/login.html" access="anon"/>59 <intercept-url pattern="/login" access="anon"/>60 <intercept-url pattern="/signin" access="anon"/>61 <intercept-url pattern="/logout" access="anon"/>62 <intercept-url pattern="/captcha/build" access="anon"/>63 <intercept-url pattern="/err/**" access="anon"/>64 <intercept-url pattern="/**" access="authc"/>65 </filters>66 </secure>67 </environment>68 </environments>69</configuration>70` ...

tmpl_new_proj_web_conf_secure.go

Source:tmpl_new_proj_web_conf_secure.go

1// Copyright 2014 The goyy Authors. All rights reserved.2// Use of this source code is governed by a MIT-style3// license that can be found in the LICENSE file.45package main67var tmplNewProjWebConfSecure = `<?xml version="1.0" encoding="UTF-8" ?>8<!DOCTYPE env PUBLIC "-//GOYY//DTD ENV 1.0//EN" "http://goyy.org/dtd/env-1.0.dtd">9<configuration>10 <environments default="development">11 <environment id="development">12 <secure name="<%.NewProjName%>">13 <enable>true</enable>14 <login-url>/login.html</login-url>15 <forbid-url>/err/403.html</forbid-url>16 <success-url>/home.html</success-url>17 <filters>18 <intercept-url pattern="/apis/sys/**/(save|saved)" access="forbid"/>19 <intercept-url pattern="/apis/**/(disable|disabled)" access="forbid"/>20 <intercept-url pattern="/apis/sys/menu/**" access="forbid"/>21 <intercept-url pattern="/apis/sys/post/**" access="forbid"/>22 <intercept-url pattern="/apis/sys/user/role/**" access="forbid"/>23 <intercept-url pattern="/apis/sys/user/(index|show|add|edit|repwd)" access="forbid"/>24 <intercept-url pattern="/**" access="anon"/>25 </filters>26 </secure>27 </environment>28 <environment id="test">29 <secure name="<%.NewProjName%>">30 <enable>true</enable>31 <login-url>/login.html</login-url>32 <forbid-url>/err/403.html</forbid-url>33 <success-url>/home.html</success-url>34 <filters>35 <intercept-url pattern="/apis/sys/**/(save|saved)" access="forbid"/>36 <intercept-url pattern="/apis/**/(disable|disabled)" access="forbid"/>37 <intercept-url pattern="/apis/sys/menu/**" access="forbid"/>38 <intercept-url pattern="/apis/sys/post/**" access="forbid"/>39 <intercept-url pattern="/apis/sys/user/role/**" access="forbid"/>40 <intercept-url pattern="/apis/sys/user/(index|show|add|edit|repwd)" access="forbid"/>41 <intercept-url pattern="/**" access="anon"/>42 </filters>43 </secure>44 </environment>45 <environment id="production">46 <secure name="<%.NewProjName%>">47 <enable>true</enable>48 <login-url>/login.html</login-url>49 <forbid-url>/err/403.html</forbid-url>50 <success-url>/home.html</success-url>51 <filters>52 <intercept-url pattern="/apis/sys/**/(save|saved)" access="forbid"/>53 <intercept-url pattern="/apis/**/(disable|disabled)" access="forbid"/>54 <intercept-url pattern="/apis/sys/menu/**" access="forbid"/>55 <intercept-url pattern="/apis/sys/post/**" access="forbid"/>56 <intercept-url pattern="/apis/sys/user/role/**" access="forbid"/>57 <intercept-url pattern="/apis/sys/user/(index|show|add|edit|repwd)" access="forbid"/>58 <intercept-url pattern="/**" access="anon"/>59 </filters>60 </secure>61 </environment>62 </environments>63</configuration>64` ...

Pattern

Using AI Code Generation

1import (2func main() {3 doc, err := html.Parse(os.Stdin)4 if err != nil {5 fmt.Fprintf(os.Stderr, "findlinks1: %v\n", err)6 os.Exit(1)7 }8 for _, link := range visit(nil, doc) {9 fmt.Println(link)10 }11}12func visit(links []string, n *html.Node) []string {13 if n.Type == html.ElementNode && n.Data == "a" {14 for _, a := range n.Attr {15 if a.Key == "href" {16 links = append(links, a.Val)17 }18 }19 }20 for c := n.FirstChild; c != nil; c = c.NextSibling {21 links = visit(links, c)22 }23}

Pattern

Using AI Code Generation

1import (2func main() {3 doc, err := html.Parse(os.Stdin)4 if err != nil {5 fmt.Fprintf(os.Stderr, "findlinks1: %v\n", err)6 os.Exit(1)7 }8 for _, link := range visit(nil, doc) {9 fmt.Println(link)10 }11}12func visit(links []string, n *html.Node) []string {13 if n.Type == html.ElementNode && n.Data == "a" {14 for _, a := range n.Attr {15 if a.Key == "href" {16 links = append(links, a.Val)17 }18 }19 }20 for c := n.FirstChild; c != nil; c = c.NextSibling {21 links = visit(links, c)22 }23}

Pattern

Using AI Code Generation

1import (2func main() {3 doc, err := html.Parse(os.Stdin)4 if err != nil {5 fmt.Fprintf(os.Stderr, "findlinks:%v\n", err)6 os.Exit(1)7 }8 for _, link := range visit(nil, doc) {9 fmt.Println(link))10}

Pattern

Using AI Code Generation

1import (2func main() {3 fmt.Println(html.EscapeString("This is <b>HTML</b>"))4 }5}6func visit(links []string, n *html.Node) []string {7 }8 }9 }10 }11}

Pattern

Using AI Code Generation

1import (2func main() {3 http.HandleFunc("/", index)4 http.HandleFunc("/process", process)5 http.Handle("/favicon.ico", http.NotFoundHandler())6 http.ListenAndServe(":8080", nil)7}8func index(w http.ResponseWriter, req *http.Request) {9 fmt.Fprintln(w, `<form method="POST" action="/process">10}11func process(w http.ResponseWriter, req *http.Request) {12 v := req.FormValue("q")13 v = strings.ToLower(v)14 fmt.Fprintln(w, "Method:", m)15 fmt.Fprintln(w, "You searched for:", v)16}

Pattern

Using AI Code Generation

1import (2func main() {3 fmt.Print("Enter the url: ")4 fmt.Scan(&url)5 log.Fatal("Invalid url")6 }7 resp, err := http.Get(url)8 if err != nil {9 log.Fatal(err)10 }11 if resp.StatusCode != http.StatusOK {12 resp.Body.Close()13 log.Fatal("Invalid url")14 }15 doc, err := html.Parse(resp.Body)16 resp.Body.Close()17 if err != nil {18 log.Fatal(err)19 }20 links := make(map[string]bool)21 getLinks(links, doc)22 for link := range links {23 fmt.Println(link)24 }25}26}

Pattern

Using AI Code Generation

1import (2func main() {3 for _, url := range os.Args[1:] {4 links, err := getLinks(url)5 if err != nil {6 fmt.Fprintf(os.Stderr, "getlinks: %v\n", err)7 }8 for _, link := range links {9 fmt.Println(link)10 }11 }12}13func getLinks(url string) ([]string, error) {14 resp, err := http.Get(url)15 if err != nil {16 return nil, fmt.Errorf("getlinks: %v\n", err)17 }18 if resp.StatusCode != http.StatusOK {19 resp.Body.Close()20 return nil, fmt.Errorf("getlinks: %s\n", resp.Status)21 }22 doc, err := html.Parse(resp.Body)23 resp.Body.Close()24 if err != nil {25 return nil, fmt.Errorf("getlinks: parsing %s as HTML: %v\n", url, err)26 }27 return visit(nil, doc), nil28}29func visit(links []string, n *html.Node) []string {30 if n.Type == html.ElementNode && n.Data == "a" {31 for _, a := range n.Attr {32 if a.Key == "href" {33 links = append(links, a.Val)34 }35 }36 }37 for c := n.FirstChild; c != nil; c = c.NextSibling {38 links = visit(links, c)39 }40func getLinks(links map[string]bool, n *html.Node) {41 if n.Type == html.ElementNode {42 if n.Data == "a" {43 for _, a := range n.Attr {44 if a.Key == "href" {45 }46 }47 }48 }49 for c := n.FirstChild; c != nil; c = c.NextSibling {50 getLinks(links, c)51 }52}

Pattern

Using AI Code Generation

1import (2func main() {3 res, err := http.Get(url)4 if err != nil {5 panic(err)6 }7 defer res.Body.Close()8 robots, err := ioutil.ReadAll(res.Body)9 if err != nil {10 panic(err)11 }12 str := string(robots)13 fmt.Println(strings.Replace(str, "Go", "golang", -1))14}

Pattern

Using AI Code Generation

1import (2func main() {3 r := strings.NewReader(`4 doc, err := html.Parse(r)5 if err != nil {6 log.Fatal(err)7 }8 matches := htmlutils.FindByID(doc, "hello")9 if len(matches) > 0 {10 fmt.Printf("%+v\n", matches[0])11 }12}

Pattern

Using AI Code Generation

1import (2func main() {3 r := strings.NewReader(`4 doc, err := html.Parse(r)5 if err != nil {6 log.Fatal(err)7 }8 matches := htmlutils.FindByID(doc, "hello")9 if len(matches) > 0 {10 fmt.Printf("%+v\n", matches[0])11 }12}

Pattern

Using AI Code Generation

1import (2func main() {3 resp, err := http.Get(url)4 if err != nil {5 log.Fatal(err)6 }7 doc, err := html.Parse(resp.Body)8 resp.Body.Close()9 if err != nil {10 log.Fatal(err)

Pattern

Using AI Code Generation

1import (2func main() {3 fmt.Println(html}4 links := visit(nil, doc)5 for _, link := range links {6 fmt.Println(link)7 }8}9func visit(links []string, n *html.Node) []string {10 if n.Type == html.ElementNode && n.Data == "a" {11 for _, a := range n.Attr {12 if a.Key == "href" {13 links = append(links, a.Val)14 }15 }16 }17 for c := n.FirstChild; c != nil; c = c.NextSibling {18 links = visit(links, c)19 }20}

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.

LambdaTest Learning Hubs:

• JUnit Tutorial
• TestNG Tutorial
• Webdriver Tutorial
• WebDriverIO Tutorial
• Protractor Tutorial
• Selenium 4 Tutorial
• Jenkins Tutorial
• NUnit Tutorial
• Jest Tutorial
• Playwright Tutorial
• Cypress Tutorial
• PyTest Tutorial

YouTube

You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.

Run K6 automation tests on LambdaTest cloud grid

Perform automation testing on 3000+ real desktop and mobile devices online.

View sample repo View DocumentationSign up Free

›

_

Most used method in

• TestGenElementsTestTextProperties
• TestGenElementsBoolProperties
• TestGenElementsIntProperties
• TestGenElementsNullProperties
• TestGenElementsURLProperties
• TestElements
• New
• Exports
• NewModuleInstance
• ParseHTML
• emptySelection
• buildMatcher
• varargFnCall
• adjacent
• adjacentUntil
• Add
• Find
• Closest
• Has
• Not
• Next
• NextAll
• Prev
• PrevAll
• Parent
• Parents
• Siblings
• PrevUntil
• NextUntil
• ParentsUntil
• Size
• End
• Eq
• First
• Last
• Contents
• Text
• Attr
• Html
• Val
• Children
• Each
• Filter
• Is
• Map
• Slice
• Get
• ToArray
• Index
• Data
• hrefURL
• Hash
• Host
• Hostname
• Port
• Username
• Password
• Origin
• Pathname
• Protocol
• RelList
• Search
• Form
• formOrElemAttr
• FormAction
• FormEnctype
• FormMethod
• FormNoValidate
• FormTarget
• Labels
• Name
• Value
• Width
• Height
• Options
• Type
• Elements
• Validity
• Length
• Method
• List
• Control
• Areas
• Images
• Disabled
• Label
• DefaultValue
• Max
• calcProgress
• Position
• SelectedIndex
• SelectedOptions
• FirstChild
• Caption
• THead
• TFoot
• Rows
• TBodies
• CellIndex
• Cells
• RowIndex
• SectionRowIndex
• Span
• TextTracks
• TestSerialize
• selToElement
• Download
• ReferrerPolicy
• Rel
• Href
• Target
• AccessKey
• Hreflang
• ToString
• Autoplay
• Controls
• Loop
• Muted
• Preload
• Src
• CrossOrigin
• CurrentSrc
• DefaultMuted
• MediaGroup
• Autofocus
• TabIndex
• Action
• Enctype
• Encoding
• AcceptCharset
• Autocomplete
• NoValidate
• Allowfullscreen
• Sizes
• Srcset
• Alt
• IsMap
• UseMap
• Required
• Checked
• DefaultChecked
• Accept
• MaxLength
• Pattern
• Placeholder
• ReadOnly
• Min
• DirName
• Multiple
• Step
• Challenge
• Keytype
• HtmlFor
• Media
• Content
• HttpEquiv
• High
• Low
• Optimum
• Cite
• Datetime
• TypeMustMatch
• Reversed
• Start
• DefaultSelected
• Selected
• Charset
• Async
• Defer
• NoModule
• KeySystem
• Sortable
• ColSpan
• RowSpan
• Headers
• Abbr
• Scope
• Sorted
• Cols
• Autocapitalize
• Wrap
• Kind
• Srclang
• Default
• getTestModuleInstance
• getTestRuntimeAndModuleInstanceWithDoc
• getTestRuntimeWithDoc
• TestParseHTML
• SerializeArray
• SerializeObject
• Serialize
• attrAsString
• resolveURL
• attrAsURL
• attrAsURLString
• attrAsInt
• attrIsPresent
• ownerFormSel
• ownerFormVal
• elemLabels
• splitAttr
• idOrNameAttr
• Prefix
• namespaceURI
• LocalName
• GetAttribute
• GetAttributeNode
• HasAttribute
• HasAttributes
• Attributes
• HasChildNodes
• TextContent
• Id
• IsEqualNode
• IsSameNode
• GetElementsByClassName
• GetElementsByTagName
• QuerySelector
• QuerySelectorAll
• NodeName
• LastChild
• FirstElementChild
• LastElementChild
• PreviousSibling
• NextSibling
• PreviousElementSibling
• NextElementSibling
• ParentNode
• ParentElement
• ChildNodes
• ChildElementCount
• ClassList
• ClassName
• Lang
• OwnerDocument
• IsDefaultNamespace
• getOwnerDocNode
• InnerHTML
• NodeType
• NodeValue
• Contains
• Matches
• TestElement
• attrToProperty
• propertyToAttr
• valueOrHTML
• getHTMLAttr
• elemList
• nodeToElement
• toNumeric
• convertDataAttrVal