adding headless uas

ua-parser · Dec 3, 2016 · 168b808 · 168b808
1 parent 118dd87
commit 168b808
Show file tree

Hide file tree

Showing 3 changed files with 160 additions and 67 deletions.
diff --git a/regexes.yaml b/regexes.yaml
@@ -349,7 +349,7 @@ user_agent_parsers:
   - regex: '\b(Dolphin)(?: |HDCN/|/INT\-)(\d+)\.(\d+)\.?(\d+)?'
 
   # Browser/major_version.minor_version
-  - regex: '(bingbot|Bolt|AdobeAIR|Jasmine|IceCat|Skyfire|Midori|Maxthon|Lynx|Arora|IBrowse|Dillo|Camino|Shiira|Fennec|Phoenix|Flock|Netscape|Lunascape|Epiphany|WebPilot|Opera Mini|Opera|NetFront|Netfront|Konqueror|Googlebot|SeaMonkey|Kazehakase|Vienna|Iceape|Iceweasel|IceWeasel|Iron|K-Meleon|Sleipnir|Galeon|GranParadiso|iCab|iTunes|MacAppStore|NetNewsWire|Space Bison|Stainless|Orca|Dolfin|BOLT|Minimo|Tizen Browser|Polaris|Abrowser|Planetweb|ICE Browser|mDolphin|qutebrowser|Otter|QupZilla|MailBar|kmail2|YahooMobileMail|ExchangeWebServices|ExchangeServicesClient|Microsoft-CryptoAPI)/(\d+)\.(\d+)(?:\.(\d+))?'
+  - regex: '(bingbot|Bolt|AdobeAIR|Jasmine|IceCat|Skyfire|Midori|Maxthon|Lynx|Arora|IBrowse|Dillo|Camino|Shiira|Fennec|Phoenix|Flock|Netscape|Lunascape|Epiphany|WebPilot|Opera Mini|Opera|NetFront|Netfront|Konqueror|Googlebot|SeaMonkey|Kazehakase|Vienna|Iceape|Iceweasel|IceWeasel|Iron|K-Meleon|Sleipnir|Galeon|GranParadiso|iCab|iTunes|MacAppStore|NetNewsWire|Space Bison|Stainless|Orca|Dolfin|BOLT|Minimo|Tizen Browser|Polaris|Abrowser|Planetweb|ICE Browser|mDolphin|qutebrowser|Otter|QupZilla|MailBar|kmail2|YahooMobileMail|ExchangeWebServices|ExchangeServicesClient)/(\d+)\.(\d+)(?:\.(\d+))?'
 
   # Chrome/Chromium/major_version.minor_version
   - regex: '(Chromium|Chrome)/(\d+)\.(\d+)(?:\.(\d+))?'
@@ -561,6 +561,9 @@ user_agent_parsers:
   - regex: '(python-requests)/(\d+)\.(\d+)'
     family_replacement: 'Python Requests'
 
+  # headless user-agents
+  - regex: '\b(Windows-Update-Agent|Microsoft-CryptoAPI|SophosUpdateManager|SophosAgent|Debian APT-HTTP|Ubuntu APT-HTTP|libcurl-agent|libwww-perl|urlgrabber|curl|Wget|OpenBSD ftp|jupdate)(?:[ /](\d+)(?:\.(\d+)(?:\.(\d+))?)?)?'
+
   - regex: '(Java)[/ ]{0,1}\d+\.(\d+)\.(\d+)[_-]*([a-zA-Z0-9]+)*'
 
   # Roku Digital-Video-Players https://www.roku.com/

diff --git a/test_resources/pgts_browser_list.yaml b/test_resources/pgts_browser_list.yaml
@@ -1051,70 +1051,70 @@ test_cases:
     patch: '0'
 
   - user_agent_string: 'curl/7.10.2 (powerpc-apple-darwin7.0) libcurl/7.10.2 OpenSSL/0.9.7b zlib/1.1.4'
-    family: 'Other'
-    major:
-    minor:
-    patch:
+    family: 'curl'
+    major: '7'
+    minor: '10'
+    patch: '2'
 
   - user_agent_string: 'curl/7.10.6 (i386-redhat-linux-gnu) libcurl/7.10.6 OpenSSL/0.9.7a ipv6 zlib/1.1.4'
-    family: 'Other'
-    major:
-    minor:
-    patch:
+    family: 'curl'
+    major: '7'
+    minor: '10'
+    patch: '6'
 
   - user_agent_string: 'curl/7.10.6 (i386-redhat-linux-gnu) libcurl/7.10.6 OpenSSL/0.9.7a ipv6 zlib/1.2.0.7'
-    family: 'Other'
-    major:
-    minor:
-    patch:
+    family: 'curl'
+    major: '7'
+    minor: '10'
+    patch: '6'
 
   - user_agent_string: 'curl/7.11.2 (i686-pc-linux-gnu) libcurl/7.10.2 OpenSSL/0.9.6i ipv6 zlib/1.1.4'
-    family: 'Other'
-    major:
-    minor:
-    patch:
+    family: 'curl'
+    major: '7'
+    minor: '11'
+    patch: '2'
 
   - user_agent_string: 'curl/7.12.0 (i686-pc-linux-gnu) libcurl/7.12.0 OpenSSL/0.9.7e ipv6 zlib/1.2.2'
-    family: 'Other'
-    major:
-    minor:
-    patch:
+    family: 'curl'
+    major: '7'
+    minor: '12'
+    patch: '0'
 
   - user_agent_string: 'curl/7.7.2 (powerpc-apple-darwin6.0) libcurl 7.7.2 (OpenSSL 0.9.6b)'
-    family: 'Other'
-    major:
-    minor:
-    patch:
+    family: 'curl'
+    major: '7'
+    minor: '7'
+    patch: '2'
 
   - user_agent_string: 'curl/7.7.3 (i686-pc-linux-gnu) libcurl 7.7.3 (OpenSSL 0.9.6)'
-    family: 'Other'
-    major:
-    minor:
-    patch:
+    family: 'curl'
+    major: '7'
+    minor: '7'
+    patch: '3'
 
   - user_agent_string: 'curl/7.7.3 (win32) libcurl 7.7.3 (OpenSSL 0.9.6)'
-    family: 'Other'
-    major:
-    minor:
-    patch:
+    family: 'curl'
+    major: '7'
+    minor: '7'
+    patch: '3'
 
   - user_agent_string: 'curl/7.9.3 (powerpc-ibm-aix4.3.3.0) libcurl 7.9.3 (OpenSSL 0.9.6m)'
-    family: 'Other'
-    major:
-    minor:
-    patch:
+    family: 'curl'
+    major: '7'
+    minor: '9'
+    patch: '3'
 
   - user_agent_string: 'curl/7.9.5 (i386-redhat-linux-gnu) libcurl 7.9.5 (OpenSSL 0.9.6b) (ipv6 enabled)'
-    family: 'Other'
-    major:
-    minor:
-    patch:
+    family: 'curl'
+    major: '7'
+    minor: '9'
+    patch: '5'
 
   - user_agent_string: 'curl/7.9.8 (i386-redhat-linux-gnu) libcurl 7.9.8 (OpenSSL 0.9.7a) (ipv6 enabled)'
-    family: 'Other'
-    major:
-    minor:
-    patch:
+    family: 'curl'
+    major: '7'
+    minor: '9'
+    patch: '8'
 
   - user_agent_string: 'DA 5.0'
     family: 'Other'
@@ -4165,27 +4165,27 @@ test_cases:
     patch:
 
   - user_agent_string: 'libwww-perl/5.79'
-    family: 'Other'
-    major:
-    minor:
+    family: 'libwww-perl'
+    major: '5'
+    minor: '79'
     patch:
 
   - user_agent_string: 'SonyEricssonT306/R101 [Html2Wml/0.4.11 libwww-perl/5.79]'
-    family: 'Other'
-    major:
-    minor:
+    family: 'libwww-perl'
+    major: '5'
+    minor: '79'
     patch:
 
   - user_agent_string: 'testJapanequeDelicious/0.1 libwww-perl/5.803'
-    family: 'Other'
-    major:
-    minor:
+    family: 'libwww-perl'
+    major: '5'
+    minor: '803'
     patch:
 
   - user_agent_string: 'W3C-checklink/4.1 [4.14] libwww-perl/5.803'
-    family: 'Other'
-    major:
-    minor:
+    family: 'libwww-perl'
+    major: '5'
+    minor: '803'
     patch:
 
   - user_agent_string: 'Mozilla/3.0 (compatible; Linkman)'
@@ -74779,15 +74779,15 @@ test_cases:
     patch:
 
   - user_agent_string: 'Wget/1.8.2 modified'
-    family: 'Other'
-    major:
-    minor:
-    patch:
+    family: 'Wget'
+    major: '1'
+    minor: '8'
+    patch: '2'
 
   - user_agent_string: 'Wget/1.9+cvs-stable (Red Hat modified)'
-    family: 'Other'
-    major:
-    minor:
+    family: 'Wget'
+    major: '1'
+    minor: '9'
     patch:
 
   - user_agent_string: 'Mozilla/4.0 (compatible; MSIE 5.00; Window 98)'

diff --git a/tests/test_ua.yaml b/tests/test_ua.yaml
@@ -887,7 +887,7 @@ test_cases:
     major: '4'
     minor: '0'
     patch: '8'
-    
+
   - user_agent_string: 'Mozilla/4.0 (compatible; Lotus-Notes/6.0; Windows-NT)'
     family: 'Lotus Notes'
     major: '6'
@@ -1013,7 +1013,7 @@ test_cases:
     major: '8'
     minor: '7'
     patch: '0'
-    
+
   - user_agent_string: 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 UBrowser/5.7.14488.1025 Safari/537.36'
     family: 'UC Browser'
     major: '5'
@@ -5864,7 +5864,7 @@ test_cases:
     major:
     minor:
     patch:
-    
+
   - user_agent_string: 'Mozilla/5.0 (compatible; YoudaoBot-rts/1.0; http://www.youdao.com/help/webmaster/spider/; )'
     family: 'YoudaoBot'
     major:
@@ -6860,3 +6860,93 @@ test_cases:
     major: '0'
     minor: '4'
     patch:
+
+  - user_agent_string: 'curl/7.29.0'
+    family: 'curl'
+    major: '7'
+    minor: '29'
+    patch: '0'
+
+  - user_agent_string: 'Debian APT-HTTP/1.3 (1.0.1ubuntu2)'
+    family: 'Debian APT-HTTP'
+    major: '1'
+    minor: '3'
+    patch:
+
+  - user_agent_string: 'jupdate'
+    family: 'jupdate'
+    major:
+    minor:
+    patch:
+
+  - user_agent_string: 'libcurl-agent/1.0'
+    family: 'libcurl-agent'
+    major: '1'
+    minor: '0'
+    patch:
+
+  - user_agent_string: 'libwww-perl/6.05'
+    family: 'libwww-perl'
+    major: '6'
+    minor: '05'
+    patch:
+
+  - user_agent_string: 'Microsoft-CryptoAPI/6.3'
+    family: 'Microsoft-CryptoAPI'
+    major: '6'
+    minor: '3'
+    patch:
+
+  - user_agent_string: 'OpenBSD ftp'
+    family: 'OpenBSD ftp'
+    major:
+    minor:
+    patch:
+
+  - user_agent_string: 'SophosAgent/1.0 (type= spa )'
+    family: 'SophosAgent'
+    major: '1'
+    minor: '0'
+    patch:
+
+  - user_agent_string: 'SophosUpdateManager/1.5.7.50 SDDS/2.0 (u= EO2ANA123G c= 6342da15-f351-4ab7-9656-3f5f2d50885d )'
+    family: 'SophosUpdateManager'
+    major: '1'
+    minor: '5'
+    patch: '7'
+
+  - user_agent_string: 'Ubuntu APT-HTTP/1.3 (0.7.20.2ubuntu6)'
+    family: 'Ubuntu APT-HTTP'
+    major: '1'
+    minor: '3'
+    patch:
+
+  - user_agent_string: 'urlgrabber/3.10 yum/3.4.3'
+    family: 'urlgrabber'
+    major: '3'
+    minor: '10'
+    patch:
+
+  - user_agent_string: 'urlgrabber/3.9.1 yum/3.2.29'
+    family: 'urlgrabber'
+    major: '3'
+    minor: '9'
+    patch: '1'
+
+  - user_agent_string: 'Wget/1.14 (linux-gnu)'
+    family: 'Wget'
+    major: '1'
+    minor: '14'
+    patch:
+
+  - user_agent_string: 'Windows-Update-Agent/7.9.9600.17729 Client-Protocol/1.21'
+    family: 'Windows-Update-Agent'
+    major: '7'
+    minor: '9'
+    patch: '9600'
+
+  - user_agent_string: 'Windows-Update-Agent/7.9.9600.18094 Client-Protocol/1.21'
+    family: 'Windows-Update-Agent'
+    major: '7'
+    minor: '9'
+    patch: '9600'