From b5ec72cbcb14479f30ae434441b5c5f658e0c2c2 Mon Sep 17 00:00:00 2001 From: Ayumi Hamasaki Date: Wed, 7 Oct 2020 16:48:48 +0100 Subject: [PATCH 1/7] Node Fetch --- .travis.yml | 22 ++++++++++------------ data/applications-bots.php | 1 + data/regexes/applications-bots.php | 2 +- data/regexes/applications-browsers.php | 2 +- data/regexes/applications-others.php | 2 +- tests/data/bots/generic.yaml | 4 ++++ 6 files changed, 18 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index ed4b58cca..be631473c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,14 +3,12 @@ dist: trusty language: php php: - - 5.4 - - 5.5 - - 5.6 - 7.0 - 7.1 - 7.2 - 7.3 - 7.4 + # - 8.0 (Release Date: November 26, 2020) before_script: - composer self-update @@ -22,20 +20,20 @@ before_script: # Set memory limit to 2 MB - echo "memory_limit=2048M" >> ~/.phpenv/versions/$(phpenv version-name)/etc/conf.d/travis.ini - # Disable X-debug on all but PHP 5.6 - - if [ "$TRAVIS_PHP_VERSION" != "5.6" ]; then phpenv config-rm xdebug.ini; fi + # Disable X-debug on all but PHP 7.0 + - if [ "$TRAVIS_PHP_VERSION" != "7.0" ]; then phpenv config-rm xdebug.ini; fi script: - vendor/bin/phpcs --standard=PSR1,PSR2 -n src - # Run tests (all except PHP 5.6) - - if [ "$TRAVIS_PHP_VERSION" != "5.6" ]; then php bin/runner.php --show check; fi - - if [ "$TRAVIS_PHP_VERSION" != "5.6" ]; then vendor/bin/phpunit; fi + # Run tests (all except PHP 7.0) + - if [ "$TRAVIS_PHP_VERSION" != "7.0" ]; then php bin/runner.php --show check; fi + - if [ "$TRAVIS_PHP_VERSION" != "7.0" ]; then vendor/bin/phpunit; fi - # Run tests with coverage enabled (only PHP 5.6) - - if [ "$TRAVIS_PHP_VERSION" = "5.6" ]; then php bin/runner.php --coverage --show check; fi - - if [ "$TRAVIS_PHP_VERSION" = "5.6" ]; then vendor/bin/phpunit --coverage-clover phpunit.xml; fi + # Run tests with coverage enabled (only PHP 7.0) + - if [ "$TRAVIS_PHP_VERSION" = "7.0" ]; then php bin/runner.php --coverage --show check; fi + - if [ "$TRAVIS_PHP_VERSION" = "7.0" ]; then vendor/bin/phpunit --coverage-clover phpunit.xml; fi after_script: # Upload coverage to Coveralls.io - - if [ "$TRAVIS_PHP_VERSION" = "5.6" ]; then travis_retry php vendor/bin/coveralls -v; fi + - if [ "$TRAVIS_PHP_VERSION" = "7.0" ]; then travis_retry php vendor/bin/coveralls -v; fi diff --git a/data/applications-bots.php b/data/applications-bots.php index 3c3cac601..67473823b 100644 --- a/data/applications-bots.php +++ b/data/applications-bots.php @@ -299,6 +299,7 @@ [ 'name' => 'Go', 'id' => 'package', 'regexp' => '/Go [0-9\.]+ package http/u' ], [ 'name' => 'Java', 'id' => 'java', 'regexp' => '/^Java\/([0-9.]*)/u' ], + [ 'name' => 'Node Fetch', 'id' => 'nodefetch', 'regexp' => '/node-fetch\/([0-9.]*)/u' ], [ 'name' => 'Perl', 'id' => 'simple', 'regexp' => '/LWP::Simple\//u' ], [ 'name' => 'Perl', 'id' => 'libwww', 'regexp' => '/libwww-perl\//u' ], [ 'name' => 'Perl', 'id' => 'lwp', 'regexp' => '/lwp-trivial\//u' ], diff --git a/data/regexes/applications-bots.php b/data/regexes/applications-bots.php index fa8685a44..4d16144af 100644 --- a/data/regexes/applications-bots.php +++ b/data/regexes/applications-bots.php @@ -2,4 +2,4 @@ namespace WhichBrowser\Data; -Applications::$BOTS_REGEX = '/(008|360|a6|abound|muncher|adaxas|addthis|admantx|ahrefsbot|archiver|altavista|curious|cloudfront|amorank|archive|jeeves|astra|backlink|baidu|bazqux|bingbot|msnbot|msmobot|bing|blogbridge|bloglines|bloglovin|blogpulse|blogram|blogtrot|blogshares|boardreader|browsershots|bubing|buck|butterfly|bytespider|heritrix|cliqz|cloudflare|comodo|commafeed|cbot|watchdog|datasearch|daumoa|digg|domaincrawl|discord|domain|exabot|exactseek|ezooms|facebook|fast|flamingo|fastladder|feed|wrangler|validator|fever|florienzh|friendica|robot|genieo|fetchor|google|gomez|ichiro|gooblog|goorss|grammarly|grub|hatena|capture|heureka|htdig|httpmon|hubpages|spider|crawler|kakao|kouio|larbin|linkedin|linkdex|livedoor|ltx71|magpie|mapion|mail\.ru|socialstreams|mixi|mnogo|monitor|dotbot|rogerbot|yeti|netcraft|netvibes|newsblur|newsgator|nimbostratus|orange|pages|psbot|pingdom|pinterest|pocket|postrank|comment|rssbar|hunter|quora|safesearch|scrapy|semrush|seznam|shopwiki|reader|simplepie|site24|slack|sogou|sophora|soso|spdycheck|spinn|detector|summify|telegram|twisted|twitter|tiny|tlsprober|typhoeus|vagabondo|voat|voila|vocus|mcrawler|w3bot|jigsaw|w3c|wayback|webindexer|wordpress|mechanize|xerka|sitemap|yacy|yandex|yahoo|y\!j|slurp|yelp|httpclient|nutch|synapse|indy|wget|curl|package|java|simple|libwww|lwp|urllib|requests|php|pear|zend|ruby)/i'; +Applications::$BOTS_REGEX = '/(008|360|a6|abound|muncher|adaxas|addthis|admantx|ahrefsbot|archiver|altavista|curious|cloudfront|amorank|archive|jeeves|astra|backlink|baidu|bazqux|bingbot|msnbot|msmobot|bing|blogbridge|bloglines|bloglovin|blogpulse|blogram|blogtrot|blogshares|boardreader|browsershots|bubing|buck|butterfly|bytespider|heritrix|cliqz|cloudflare|comodo|commafeed|cbot|watchdog|datasearch|daumoa|digg|domaincrawl|discord|domain|exabot|exactseek|ezooms|facebook|fast|flamingo|fastladder|feed|wrangler|validator|fever|florienzh|friendica|robot|genieo|fetchor|google|gomez|ichiro|gooblog|goorss|grammarly|grub|hatena|capture|heureka|htdig|httpmon|hubpages|spider|crawler|kakao|kouio|larbin|linkedin|linkdex|livedoor|ltx71|magpie|mapion|mail\.ru|socialstreams|mixi|mnogo|monitor|dotbot|rogerbot|yeti|netcraft|netvibes|newsblur|newsgator|nimbostratus|orange|pages|psbot|pingdom|pinterest|pocket|postrank|comment|rssbar|hunter|quora|safesearch|scrapy|semrush|seznam|shopwiki|reader|simplepie|site24|slack|sogou|sophora|soso|spdycheck|spinn|detector|summify|telegram|twisted|twitter|tiny|tlsprober|typhoeus|vagabondo|voat|voila|vocus|mcrawler|w3bot|jigsaw|w3c|wayback|webindexer|wordpress|mechanize|xerka|sitemap|yacy|yandex|yahoo|y\!j|slurp|yelp|httpclient|nutch|synapse|indy|wget|curl|package|java|nodefetch|simple|libwww|lwp|urllib|requests|php|pear|zend|ruby)/i'; diff --git a/data/regexes/applications-browsers.php b/data/regexes/applications-browsers.php index 4ffa37fb7..56d6f4a30 100644 --- a/data/regexes/applications-browsers.php +++ b/data/regexes/applications-browsers.php @@ -2,4 +2,4 @@ namespace WhichBrowser\Data; -Applications::$BROWSERS_REGEX = '/(115|126|2345|37abc|qihu|1stbrowser|7star|abrowse|acorn|acoo|amigo|aol|america|apus|arachne|arora|aweb|avant|asw|avg|aviator|avira|flyflow|bdbrowser|bdspark|basilisk|beamrise|beonex|blackwren|blackhawk|bolt|blackbird|brave|brisk|browsex|browzar|byffox|camino|canure|ccleaner|cent|charon|chedot|cheshire|chimlac|chimera|chromeplus|colibri|columbus|cometbird|comodo|dragon|conkeror|coc|coolnovo|corom|crazy|crusta|cunaguaro|cyberdog|cyberfox|daedalus|degan|demeter|deskbrowse|diigo|dillo|doczilla|rhodonit|dplus|elements|eudoraweb|falkon|flamesky|flock|framafox|freeu|galeon|globalmojo|epiphany|greenbrowser|hola|hydra|ibrowse|icab|iceape|icecat|icedove|iceweasel|irider|iridium|iron|kapiko|kazehakase|kchrome|kinza|kkman|klondike|kylo|meleon|lbrowser|lbbrowser|light|lobo|lolifox|lovense|lunascape|naenara|netcaptor|madfox|mxbrowser|min|mini|mixshark|mlbrowser|mypal|multixp|netpositive|nichrome|origyn|owb|live|juzi|orca|oregano|otter|palemoon|patriott|perk|polarity|privacy|qiyu|qtweb|quicklook|qupzilla|qute|rekonq|ryouko|sylera|saayaa|salam|shiira|sielo|sitekiosk|sleipnir|slimjet|metasr|sparrow|stainless|station|sunchrome|sundance|sundial|superbird|swiftweasel|swiftfox|swing|to-browser|tencent|tenfourfox|theworld|tulip|tungsten|ultrab|usejump|vimprobable|vivaldi|airwatch|vonkeror|voyager|waterfox|whale|winwap|xvast|yabrowser|zvu|zipzap|1password|3g explorer|4g explorer|aloha|atomiclite|au|awakening|baidu|bdmobile|bidu|baiduboxapp|browser|blazer|acheetahi|cornowser|coolmarket|cute|digia|doris|doroty|www browser|epic|eui|go\.web|helium|heytap|huohoubrowser|surfboard|isurf|jig browser|jumanji|tnsbrowser|kiosk|kiwi|lebrowser|liebao|macross|mammoth|mcent|mint|mib|cmcs|motorola|multizilla|netfrontlife|onebrowser|oneplus|plink|polaris|pure|openwave|up|km|oppo|quark|realme|semc|stnc|sogou|symphony|tenta|ucpro|vivo|weblite|wk|yaani|alti|aplix|awox|browserserver|isis|netbox|spyglass|oregan|viera|zetakey|bunjalloo|nwf|nuanti|wib|prince|demobrowser|dooble|dwb|evm|fireweb|hive|intermec|phantom|qt|roccat|raven|rockmelt|scope|slim|smbrowser|snowshoe|sputnik|surf|tao|taz|uzbl|villanova|vimb|wada|wavelink|webrender|webster|wyzo|yelang|yrcweb|linemode|lynx|links|w3m|nimbus|Webwasher|sankore|adapi|brightsign|coship)/i'; +Applications::$BROWSERS_REGEX = '/(115|126|2345|37abc|qihu|1stbrowser|7star|abrowse|acorn|acoo|amigo|aol|america|apus|arachne|arora|aweb|avant|asw|avg|aviator|avira|flyflow|bdbrowser|bdspark|basilisk|beamrise|beonex|blackwren|blackhawk|bolt|blackbird|brave|brisk|browsex|browzar|byffox|camino|canure|ccleaner|cent|charon|chedot|cheshire|chimlac|chimera|chromeplus|colibri|columbus|cometbird|comodo|dragon|conkeror|coc|coolnovo|corom|crazy|crusta|cunaguaro|cyberdog|cyberfox|daedalus|degan|demeter|deskbrowse|diigo|dillo|doczilla|rhodonit|dplus|elements|eudoraweb|falkon|flamesky|flock|framafox|freeu|galeon|globalmojo|epiphany|greenbrowser|hola|hydra|ibrowse|icab|iceape|icecat|icedove|iceweasel|irider|iridium|iron|kapiko|kazehakase|kchrome|kinza|kkman|klondike|kylo|meleon|lbrowser|lbbrowser|light|lobo|lolifox|lovense|lunascape|naenara|netcaptor|madfox|mxbrowser|min|mini|mixshark|mlbrowser|mypal|multixp|netpositive|nichrome|origyn|owb|live|juzi|orca|oregano|otter|palemoon|patriott|perk|polarity|privacy|qiyu|qtweb|quicklook|qupzilla|qute|rekonq|ryouko|sylera|saayaa|salam|shiira|sielo|sitekiosk|sleipnir|slimjet|metasr|sparrow|stainless|station|sunchrome|sundance|sundial|superbird|swiftweasel|swiftfox|swing|to-browser|tencent|tenfourfox|theworld|tulip|tungsten|ultrab|usejump|vimprobable|vivaldi|airwatch|vonkeror|voyager|waterfox|whale|winwap|xvast|yabrowser|zvu|zipzap|1password|3g explorer|4g explorer|aloha|atomiclite|au|awakening|baidu|bdmobile|bidu|baiduboxapp|browser|blazer|acheetahi|cornowser|coolmarket|cute|digia|doris|doroty|ecosia|www browser|epic|eui|go\.web|helium|heytap|huohoubrowser|surfboard|isurf|jig browser|jumanji|tnsbrowser|kiosk|kiwi|lebrowser|liebao|macross|mammoth|mcent|mint|mobicip|mib|cmcs|motorola|multizilla|netfrontlife|onebrowser|oneplus|plink|polaris|pure|openwave|up|km|oppo|quark|realme|semc|stnc|sogou|symphony|tenta|ucpro|vivo|weblite|wk|yaani|alti|aplix|awox|browserserver|isis|netbox|spyglass|oregan|viera|zetakey|bunjalloo|nwf|nuanti|wib|prince|demobrowser|dooble|dwb|evm|fireweb|hive|intermec|phantom|qt|roccat|raven|rockmelt|scope|slim|smbrowser|snowshoe|sputnik|surf|tao|taz|uzbl|villanova|vimb|wada|wavelink|webrender|webster|wyzo|yelang|yrcweb|linemode|lynx|links|w3m|nimbus|Webwasher|sankore|adapi|brightsign|coship)/i'; diff --git a/data/regexes/applications-others.php b/data/regexes/applications-others.php index c8000539e..483dcade1 100644 --- a/data/regexes/applications-others.php +++ b/data/regexes/applications-others.php @@ -2,4 +2,4 @@ namespace WhichBrowser\Data; -Applications::$OTHERS_REGEX = '/(itunes|qt|bluefish|nightingale|songbird|stagefright|substream|vlc|windows-media|coreplayer|flycast|boxee|kodi|xbmc|lightning|thunderbird|outlook|lotus|postbox|bat|yahoo|daum|flipboard|akregator|blogos|cococ|feed|liferea|news|jetbrains|rss|reeder|reedkit|rome|ziepod|messenger|kakao|kik|line|slack|viber|whatsapp|wire|yammer|fbios|fb4a|googleplus|instagram|pinterest|weibo|tumblr|twitter|wp-android|yelp|office|bingweb|hao123|gsa|naver|sogousearch|yandex|atom|golive|brackets|iweb|frontpage|amaya|websphere|download|tansodl|origin|secondlife|valve|raptr|alipay|cooliris|google|leechcraft|expeditor)/i'; +Applications::$OTHERS_REGEX = '/(itunes|qt|bluefish|nightingale|songbird|stagefright|substream|vlc|windows-media|coreplayer|flycast|boxee|kodi|xbmc|lightning|thunderbird|outlook|lotus|postbox|bat|yahoo|daum|flipboard|akregator|blogos|cococ|feed|liferea|news|jetbrains|rss|reeder|reedkit|rome|ziepod|messenger|kakao|kik|line|slack|viber|whatsapp|wire|yammer|zalo|fbios|fb4a|googleplus|instagram|pinterest|weibo|tumblr|twitter|wp-android|yelp|office|bingweb|hao123|gsa|naver|sogousearch|yandex|atom|golive|brackets|iweb|frontpage|amaya|websphere|download|tansodl|origin|secondlife|valve|raptr|alipay|cooliris|google|leechcraft|expeditor)/i'; diff --git a/tests/data/bots/generic.yaml b/tests/data/bots/generic.yaml index e219a037e..4c7056198 100644 --- a/tests/data/bots/generic.yaml +++ b/tests/data/bots/generic.yaml @@ -586,3 +586,7 @@ headers: 'User-Agent: PocketParser/2.0 (+https://getpocket.com/pocketparser_ua)' readable: 'Pocket 2.0' result: { browser: { name: Pocket, version: '2.0' }, device: { type: bot } } +- + headers: 'User-Agent: node-fetch/1.0 (+https://github.com/bitinn/node-fetch)' + readable: 'Node Fetch 1.0' + result: { browser: { name: 'Node Fetch', version: '1.0' }, device: { type: bot } } From 798f68a7ed164da697f75b36a10933c8f3cfd4d8 Mon Sep 17 00:00:00 2001 From: Ayumi Hamasaki Date: Wed, 7 Oct 2020 17:08:12 +0100 Subject: [PATCH 2/7] move up the list --- data/applications-bots.php | 2 +- data/regexes/applications-bots.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/data/applications-bots.php b/data/applications-bots.php index 67473823b..b07cc0775 100644 --- a/data/applications-bots.php +++ b/data/applications-bots.php @@ -173,6 +173,7 @@ [ 'name' => 'NewsGator', 'id' => 'newsgator', 'regexp' => '/NewsGatorOnline\/([0-9.]*)/u' ], [ 'name' => 'NewsGator FetchLinks', 'id' => 'newsgator', 'regexp' => '/NewsGator FetchLinks extension\/([0-9.]*)/u' ], [ 'name' => 'Nimbostratus', 'id' => 'nimbostratus', 'regexp' => '/Nimbostratus-Bot\/v([0-9.]*)/u' ], + [ 'name' => 'Node Fetch', 'id' => 'nodefetch', 'regexp' => '/node-fetch\/([0-9.]*)/u' ], [ 'name' => 'OpenWebSpider', 'id' => 'spider', 'regexp' => '/OpenWebSpider v([0-9.]*)/u' ], [ 'name' => 'OrangeBot', 'id' => 'orange', 'regexp' => '/OrangeBot-Collector\/([0-9.]*)/u' ], [ 'name' => 'OrangeBot', 'id' => 'orange', 'regexp' => '/OrangeBot-Mobile/u' ], @@ -299,7 +300,6 @@ [ 'name' => 'Go', 'id' => 'package', 'regexp' => '/Go [0-9\.]+ package http/u' ], [ 'name' => 'Java', 'id' => 'java', 'regexp' => '/^Java\/([0-9.]*)/u' ], - [ 'name' => 'Node Fetch', 'id' => 'nodefetch', 'regexp' => '/node-fetch\/([0-9.]*)/u' ], [ 'name' => 'Perl', 'id' => 'simple', 'regexp' => '/LWP::Simple\//u' ], [ 'name' => 'Perl', 'id' => 'libwww', 'regexp' => '/libwww-perl\//u' ], [ 'name' => 'Perl', 'id' => 'lwp', 'regexp' => '/lwp-trivial\//u' ], diff --git a/data/regexes/applications-bots.php b/data/regexes/applications-bots.php index 4d16144af..271183787 100644 --- a/data/regexes/applications-bots.php +++ b/data/regexes/applications-bots.php @@ -2,4 +2,4 @@ namespace WhichBrowser\Data; -Applications::$BOTS_REGEX = '/(008|360|a6|abound|muncher|adaxas|addthis|admantx|ahrefsbot|archiver|altavista|curious|cloudfront|amorank|archive|jeeves|astra|backlink|baidu|bazqux|bingbot|msnbot|msmobot|bing|blogbridge|bloglines|bloglovin|blogpulse|blogram|blogtrot|blogshares|boardreader|browsershots|bubing|buck|butterfly|bytespider|heritrix|cliqz|cloudflare|comodo|commafeed|cbot|watchdog|datasearch|daumoa|digg|domaincrawl|discord|domain|exabot|exactseek|ezooms|facebook|fast|flamingo|fastladder|feed|wrangler|validator|fever|florienzh|friendica|robot|genieo|fetchor|google|gomez|ichiro|gooblog|goorss|grammarly|grub|hatena|capture|heureka|htdig|httpmon|hubpages|spider|crawler|kakao|kouio|larbin|linkedin|linkdex|livedoor|ltx71|magpie|mapion|mail\.ru|socialstreams|mixi|mnogo|monitor|dotbot|rogerbot|yeti|netcraft|netvibes|newsblur|newsgator|nimbostratus|orange|pages|psbot|pingdom|pinterest|pocket|postrank|comment|rssbar|hunter|quora|safesearch|scrapy|semrush|seznam|shopwiki|reader|simplepie|site24|slack|sogou|sophora|soso|spdycheck|spinn|detector|summify|telegram|twisted|twitter|tiny|tlsprober|typhoeus|vagabondo|voat|voila|vocus|mcrawler|w3bot|jigsaw|w3c|wayback|webindexer|wordpress|mechanize|xerka|sitemap|yacy|yandex|yahoo|y\!j|slurp|yelp|httpclient|nutch|synapse|indy|wget|curl|package|java|nodefetch|simple|libwww|lwp|urllib|requests|php|pear|zend|ruby)/i'; +Applications::$BOTS_REGEX = '/(008|360|a6|abound|muncher|adaxas|addthis|admantx|ahrefsbot|archiver|altavista|curious|cloudfront|amorank|archive|jeeves|astra|backlink|baidu|bazqux|bingbot|msnbot|msmobot|bing|blogbridge|bloglines|bloglovin|blogpulse|blogram|blogtrot|blogshares|boardreader|browsershots|bubing|buck|butterfly|bytespider|heritrix|cliqz|cloudflare|comodo|commafeed|cbot|watchdog|datasearch|daumoa|digg|domaincrawl|discord|domain|exabot|exactseek|ezooms|facebook|fast|flamingo|fastladder|feed|wrangler|validator|fever|florienzh|friendica|robot|genieo|fetchor|google|gomez|ichiro|gooblog|goorss|grammarly|grub|hatena|capture|heureka|htdig|httpmon|hubpages|spider|crawler|kakao|kouio|larbin|linkedin|linkdex|livedoor|ltx71|magpie|mapion|mail\.ru|socialstreams|mixi|mnogo|monitor|dotbot|rogerbot|yeti|netcraft|netvibes|newsblur|newsgator|nodefetch|nimbostratus|orange|pages|psbot|pingdom|pinterest|pocket|postrank|comment|rssbar|hunter|quora|safesearch|scrapy|semrush|seznam|shopwiki|reader|simplepie|site24|slack|sogou|sophora|soso|spdycheck|spinn|detector|summify|telegram|twisted|twitter|tiny|tlsprober|typhoeus|vagabondo|voat|voila|vocus|mcrawler|w3bot|jigsaw|w3c|wayback|webindexer|wordpress|mechanize|xerka|sitemap|yacy|yandex|yahoo|y\!j|slurp|yelp|httpclient|nutch|synapse|indy|wget|curl|package|java|simple|libwww|lwp|urllib|requests|php|pear|zend|ruby)/i'; From f4139f77c346a4623f49046883e13749ed863bd9 Mon Sep 17 00:00:00 2001 From: Ayumi Hamasaki Date: Wed, 7 Oct 2020 17:36:52 +0100 Subject: [PATCH 3/7] more tests --- data/applications-bots.php | 2 +- data/regexes/applications-bots.php | 2 +- src/Analyser/Header/Useragent/Bot.php | 14 ++++++++++++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/data/applications-bots.php b/data/applications-bots.php index b07cc0775..67473823b 100644 --- a/data/applications-bots.php +++ b/data/applications-bots.php @@ -173,7 +173,6 @@ [ 'name' => 'NewsGator', 'id' => 'newsgator', 'regexp' => '/NewsGatorOnline\/([0-9.]*)/u' ], [ 'name' => 'NewsGator FetchLinks', 'id' => 'newsgator', 'regexp' => '/NewsGator FetchLinks extension\/([0-9.]*)/u' ], [ 'name' => 'Nimbostratus', 'id' => 'nimbostratus', 'regexp' => '/Nimbostratus-Bot\/v([0-9.]*)/u' ], - [ 'name' => 'Node Fetch', 'id' => 'nodefetch', 'regexp' => '/node-fetch\/([0-9.]*)/u' ], [ 'name' => 'OpenWebSpider', 'id' => 'spider', 'regexp' => '/OpenWebSpider v([0-9.]*)/u' ], [ 'name' => 'OrangeBot', 'id' => 'orange', 'regexp' => '/OrangeBot-Collector\/([0-9.]*)/u' ], [ 'name' => 'OrangeBot', 'id' => 'orange', 'regexp' => '/OrangeBot-Mobile/u' ], @@ -300,6 +299,7 @@ [ 'name' => 'Go', 'id' => 'package', 'regexp' => '/Go [0-9\.]+ package http/u' ], [ 'name' => 'Java', 'id' => 'java', 'regexp' => '/^Java\/([0-9.]*)/u' ], + [ 'name' => 'Node Fetch', 'id' => 'nodefetch', 'regexp' => '/node-fetch\/([0-9.]*)/u' ], [ 'name' => 'Perl', 'id' => 'simple', 'regexp' => '/LWP::Simple\//u' ], [ 'name' => 'Perl', 'id' => 'libwww', 'regexp' => '/libwww-perl\//u' ], [ 'name' => 'Perl', 'id' => 'lwp', 'regexp' => '/lwp-trivial\//u' ], diff --git a/data/regexes/applications-bots.php b/data/regexes/applications-bots.php index 271183787..4d16144af 100644 --- a/data/regexes/applications-bots.php +++ b/data/regexes/applications-bots.php @@ -2,4 +2,4 @@ namespace WhichBrowser\Data; -Applications::$BOTS_REGEX = '/(008|360|a6|abound|muncher|adaxas|addthis|admantx|ahrefsbot|archiver|altavista|curious|cloudfront|amorank|archive|jeeves|astra|backlink|baidu|bazqux|bingbot|msnbot|msmobot|bing|blogbridge|bloglines|bloglovin|blogpulse|blogram|blogtrot|blogshares|boardreader|browsershots|bubing|buck|butterfly|bytespider|heritrix|cliqz|cloudflare|comodo|commafeed|cbot|watchdog|datasearch|daumoa|digg|domaincrawl|discord|domain|exabot|exactseek|ezooms|facebook|fast|flamingo|fastladder|feed|wrangler|validator|fever|florienzh|friendica|robot|genieo|fetchor|google|gomez|ichiro|gooblog|goorss|grammarly|grub|hatena|capture|heureka|htdig|httpmon|hubpages|spider|crawler|kakao|kouio|larbin|linkedin|linkdex|livedoor|ltx71|magpie|mapion|mail\.ru|socialstreams|mixi|mnogo|monitor|dotbot|rogerbot|yeti|netcraft|netvibes|newsblur|newsgator|nodefetch|nimbostratus|orange|pages|psbot|pingdom|pinterest|pocket|postrank|comment|rssbar|hunter|quora|safesearch|scrapy|semrush|seznam|shopwiki|reader|simplepie|site24|slack|sogou|sophora|soso|spdycheck|spinn|detector|summify|telegram|twisted|twitter|tiny|tlsprober|typhoeus|vagabondo|voat|voila|vocus|mcrawler|w3bot|jigsaw|w3c|wayback|webindexer|wordpress|mechanize|xerka|sitemap|yacy|yandex|yahoo|y\!j|slurp|yelp|httpclient|nutch|synapse|indy|wget|curl|package|java|simple|libwww|lwp|urllib|requests|php|pear|zend|ruby)/i'; +Applications::$BOTS_REGEX = '/(008|360|a6|abound|muncher|adaxas|addthis|admantx|ahrefsbot|archiver|altavista|curious|cloudfront|amorank|archive|jeeves|astra|backlink|baidu|bazqux|bingbot|msnbot|msmobot|bing|blogbridge|bloglines|bloglovin|blogpulse|blogram|blogtrot|blogshares|boardreader|browsershots|bubing|buck|butterfly|bytespider|heritrix|cliqz|cloudflare|comodo|commafeed|cbot|watchdog|datasearch|daumoa|digg|domaincrawl|discord|domain|exabot|exactseek|ezooms|facebook|fast|flamingo|fastladder|feed|wrangler|validator|fever|florienzh|friendica|robot|genieo|fetchor|google|gomez|ichiro|gooblog|goorss|grammarly|grub|hatena|capture|heureka|htdig|httpmon|hubpages|spider|crawler|kakao|kouio|larbin|linkedin|linkdex|livedoor|ltx71|magpie|mapion|mail\.ru|socialstreams|mixi|mnogo|monitor|dotbot|rogerbot|yeti|netcraft|netvibes|newsblur|newsgator|nimbostratus|orange|pages|psbot|pingdom|pinterest|pocket|postrank|comment|rssbar|hunter|quora|safesearch|scrapy|semrush|seznam|shopwiki|reader|simplepie|site24|slack|sogou|sophora|soso|spdycheck|spinn|detector|summify|telegram|twisted|twitter|tiny|tlsprober|typhoeus|vagabondo|voat|voila|vocus|mcrawler|w3bot|jigsaw|w3c|wayback|webindexer|wordpress|mechanize|xerka|sitemap|yacy|yandex|yahoo|y\!j|slurp|yelp|httpclient|nutch|synapse|indy|wget|curl|package|java|nodefetch|simple|libwww|lwp|urllib|requests|php|pear|zend|ruby)/i'; diff --git a/src/Analyser/Header/Useragent/Bot.php b/src/Analyser/Header/Useragent/Bot.php index e73aee116..80786d5bd 100644 --- a/src/Analyser/Header/Useragent/Bot.php +++ b/src/Analyser/Header/Useragent/Bot.php @@ -30,6 +30,20 @@ private function &detectBot($ua) $this->data->device->type = Constants\DeviceType::BOT; } + + /* Detect Node Fetch */ + + if (preg_match('/node-fetch\/([0-9.]*)/u', $ua, $result)) { + $this->data->browser->reset(); + $this->data->os->reset(); + $this->data->engine->reset(); + $this->data->device->reset(); + + $this->data->browser->name = 'Node Fetch'; + $this->data->browser->version = $result[1] ?? ''; + + $this->data->device->type = Constants\DeviceType::BOT; + } /* Detect based on a predefined list or markers */ From 145b362f02ebdd65965ab445f1891912b7118628 Mon Sep 17 00:00:00 2001 From: Ayumi Hamasaki Date: Wed, 7 Oct 2020 17:48:24 +0100 Subject: [PATCH 4/7] Update Bot.php --- src/Analyser/Header/Useragent/Bot.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Analyser/Header/Useragent/Bot.php b/src/Analyser/Header/Useragent/Bot.php index 80786d5bd..768e790cf 100644 --- a/src/Analyser/Header/Useragent/Bot.php +++ b/src/Analyser/Header/Useragent/Bot.php @@ -40,7 +40,7 @@ private function &detectBot($ua) $this->data->device->reset(); $this->data->browser->name = 'Node Fetch'; - $this->data->browser->version = $result[1] ?? ''; + //$this->data->browser->version = $result[1] ?? ''; $this->data->device->type = Constants\DeviceType::BOT; } From 65405c7ddfcf5c8892d448431fc5756b9d65e76e Mon Sep 17 00:00:00 2001 From: Ayumi Hamasaki Date: Wed, 7 Oct 2020 18:44:29 +0100 Subject: [PATCH 5/7] Update Bot.php --- src/Analyser/Header/Useragent/Bot.php | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Analyser/Header/Useragent/Bot.php b/src/Analyser/Header/Useragent/Bot.php index 768e790cf..e79bc54e5 100644 --- a/src/Analyser/Header/Useragent/Bot.php +++ b/src/Analyser/Header/Useragent/Bot.php @@ -35,12 +35,10 @@ private function &detectBot($ua) if (preg_match('/node-fetch\/([0-9.]*)/u', $ua, $result)) { $this->data->browser->reset(); - $this->data->os->reset(); - $this->data->engine->reset(); $this->data->device->reset(); $this->data->browser->name = 'Node Fetch'; - //$this->data->browser->version = $result[1] ?? ''; + $this->data->browser->version = $result[1]; $this->data->device->type = Constants\DeviceType::BOT; } From 24ae463d3a619d382c12c3099b22ef924c76347f Mon Sep 17 00:00:00 2001 From: Ayumi Hamasaki Date: Wed, 7 Oct 2020 19:04:11 +0100 Subject: [PATCH 6/7] Update Bot.php --- src/Analyser/Header/Useragent/Bot.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Analyser/Header/Useragent/Bot.php b/src/Analyser/Header/Useragent/Bot.php index e79bc54e5..58a06d80e 100644 --- a/src/Analyser/Header/Useragent/Bot.php +++ b/src/Analyser/Header/Useragent/Bot.php @@ -33,12 +33,12 @@ private function &detectBot($ua) /* Detect Node Fetch */ - if (preg_match('/node-fetch\/([0-9.]*)/u', $ua, $result)) { + if (preg_match('/node-fetch\/([0-9.]*)/u', $ua, $match)) { $this->data->browser->reset(); $this->data->device->reset(); $this->data->browser->name = 'Node Fetch'; - $this->data->browser->version = $result[1]; + $this->data->browser->version = new Version([ 'value' => $match[1] ]); $this->data->device->type = Constants\DeviceType::BOT; } From 181fb2e5d43dac353889a941eb5622fe2c967b4d Mon Sep 17 00:00:00 2001 From: Ayumi Hamasaki Date: Wed, 7 Oct 2020 19:04:58 +0100 Subject: [PATCH 7/7] Update Bot.php --- src/Analyser/Header/Useragent/Bot.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Analyser/Header/Useragent/Bot.php b/src/Analyser/Header/Useragent/Bot.php index 58a06d80e..e353f5ee6 100644 --- a/src/Analyser/Header/Useragent/Bot.php +++ b/src/Analyser/Header/Useragent/Bot.php @@ -4,6 +4,7 @@ use WhichBrowser\Constants; use WhichBrowser\Data; +use WhichBrowser\Model\Version; trait Bot {