Merge pull request #468 from daveol/master

Add ArcGIS/GIS products

regexes.yaml

@@ -1,6 +1,50 @@
 user_agent_parsers:
   #### SPECIAL CASES TOP ####
 
+  # ESRI Server products
+  - regex: '(GeoEvent Server) (\d+)(?:\.(\d+)(?:\.(\d+)|)|)'
+
+  # ESRI ArcGIS Desktop Products
+  - regex: '(ArcGIS Pro)(?: (\d+)\.(\d+)\.([^ ]+)|)'
+
+  - regex: 'ArcGIS Client Using WinInet'
+    family_replacement: 'ArcMap'
+
+  - regex: '(OperationsDashboard)-(?:Windows)-(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Operations Dashboard for ArcGIS'
+
+  - regex: '(arcgisearth)/(\d+)\.(\d+)(?:\.(\d+)|)'
+    family_replacement: 'ArcGIS Earth'
+
+  - regex: 'com.esri.(earth).phone/(\d+)\.(\d+)(?:\.(\d+)|)'
+    family_replacement: 'ArcGIS Earth'
+
+  # ESRI ArcGIS Mobile Products
+  - regex: '(arcgis-explorer)/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Explorer for ArcGIS'
+
+  - regex: 'arcgis-(collector|aurora)/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Collector for ArcGIS'
+
+  - regex: '(arcgis-workforce)/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Workforce for ArcGIS'
+
+  - regex: '(Collector|Explorer|Workforce)-(?:Android|iOS)-(\d+)\.(\d+)(?:\.(\d+)|)'
+    family_replacement: '$1 for ArcGIS'
+
+  - regex: '(Explorer|Collector)/(\d+) CFNetwork'
+    family_replacement: '$1 for ArcGIS'
+
+  # ESRI ArcGIS Runtimes
+  - regex: 'ArcGISRuntime-(Android|iOS|NET|Qt)/(\d+)\.(\d+)(?:\.(\d+)|)'
+    family_replacement: 'ArcGIS Runtime SDK for $1'
+
+  - regex: 'ArcGIS\.?(iOS|Android|NET|Qt)(?:-|\.)(\d+)\.(\d+)(?:\.(\d+)|)'
+    family_replacement: 'ArcGIS Runtime SDK for $1'
+
+  - regex: 'ArcGIS\.Runtime\.(Qt)\.(\d+)\.(\d+)(?:\.(\d+)|)'
+    family_replacement: 'ArcGIS Runtime SDK for $1'
+
   # CFNetwork Podcast catcher Applications
   - regex: '^(Luminary)[Stage]+/(\d+) CFNetwork'
   - regex: '(ESPN)[%20| ]+Radio/(\d+)\.(\d+)\.(\d+) CFNetwork'
@@ -101,12 +145,24 @@ user_agent_parsers:
     family_replacement: 'Pinterestbot'
 
   # Bots
-  - regex: '(CSimpleSpider|Cityreview Robot|CrawlDaddy|CrawlFire|Finderbots|Index crawler|Job Roboter|KiwiStatus Spider|Lijit Crawler|QuerySeekerSpider|ScollSpider|Trends Crawler|USyd-NLP-Spider|SiteCat Webbot|BotName\/\$BotVersion|123metaspider-Bot|1470\.net crawler|50\.nu|8bo Crawler Bot|Aboundex|Accoona-[A-z]{1,30}-Agent|AdsBot-Google(?:-[a-z]{1,30}|)|altavista|AppEngine-Google|archive.{0,30}\.org_bot|archiver|Ask Jeeves|[Bb]ai[Dd]u[Ss]pider(?:-[A-Za-z]{1,30})(?:-[A-Za-z]{1,30}|)|bingbot|BingPreview|blitzbot|BlogBridge|Bloglovin|BoardReader Blog Indexer|BoardReader Favicon Fetcher|boitho.com-dc|BotSeer|BUbiNG|\b\w{0,30}favicon\w{0,30}\b|\bYeti(?:-[a-z]{1,30}|)|Catchpoint(?: bot|)|[Cc]harlotte|Checklinks|clumboot|Comodo HTTP\(S\) Crawler|Comodo-Webinspector-Crawler|ConveraCrawler|CRAWL-E|CrawlConvera|Daumoa(?:-feedfetcher|)|Feed Seeker Bot|Feedbin|findlinks|Flamingo_SearchEngine|FollowSite Bot|furlbot|Genieo|gigabot|GomezAgent|gonzo1|(?:[a-zA-Z]{1,30}-|)Googlebot(?:-[a-zA-Z]{1,30}|)|Google SketchUp|grub-client|gsa-crawler|heritrix|HiddenMarket|holmes|HooWWWer|htdig|ia_archiver|ICC-Crawler|Icarus6j|ichiro(?:/mobile|)|IconSurf|IlTrovatore(?:-Setaccio|)|InfuzApp|Innovazion Crawler|InternetArchive|IP2[a-z]{1,30}Bot|jbot\b|KaloogaBot|Kraken|Kurzor|larbin|LEIA|LesnikBot|Linguee Bot|LinkAider|LinkedInBot|Lite Bot|Llaut|lycos|Mail\.RU_Bot|masscan|masidani_bot|Mediapartners-Google|Microsoft .{0,30} Bot|mogimogi|mozDex|MJ12bot|msnbot(?:-media {0,2}|)|msrbot|Mtps Feed Aggregation System|netresearch|Netvibes|NewsGator[^/]{0,30}|^NING|Nutch[^/]{0,30}|Nymesis|ObjectsSearch|OgScrper|Orbiter|OOZBOT|PagePeeker|PagesInventory|PaxleFramework|Peeplo Screenshot Bot|PlantyNet_WebRobot|Pompos|Qwantify|Read%20Later|Reaper|RedCarpet|Retreiver|Riddler|Rival IQ|scooter|Scrapy|Scrubby|searchsight|seekbot|semanticdiscovery|SemrushBot|Simpy|SimplePie|SEOstats|SimpleRSS|SiteCon|Slackbot-LinkExpanding|Slack-ImgProxy|Slurp|snappy|Speedy Spider|Squrl Java|Stringer|TheUsefulbot|ThumbShotsBot|Thumbshots\.ru|Tiny Tiny RSS|Twitterbot|WhatsApp|URL2PNG|Vagabondo|VoilaBot|^vortex|Votay bot|^voyager|WASALive.Bot|Web-sniffer|WebThumb|WeSEE:[A-z]{1,30}|WhatWeb|WIRE|WordPress|Wotbox|www\.almaden\.ibm\.com|Xenu(?:.s|) Link Sleuth|Xerka [A-z]{1,30}Bot|yacy(?:bot|)|YahooSeeker|Yahoo! Slurp|Yandex\w{1,30}|YodaoBot(?:-[A-z]{1,30}|)|YottaaMonitor|Yowedo|^Zao|^Zao-Crawler|ZeBot_www\.ze\.bz|ZooShot|ZyBorg)(?:[ /]v?(\d+)(?:\.(\d+)(?:\.(\d+)|)|)|)'
+  - regex: '(CSimpleSpider|Cityreview Robot|CrawlDaddy|CrawlFire|Finderbots|Index crawler|Job Roboter|KiwiStatus Spider|Lijit Crawler|QuerySeekerSpider|ScollSpider|Trends Crawler|USyd-NLP-Spider|SiteCat Webbot|BotName\/\$BotVersion|123metaspider-Bot|1470\.net crawler|50\.nu|8bo Crawler Bot|Aboundex|Accoona-[A-z]{1,30}-Agent|AdsBot-Google(?:-[a-z]{1,30}|)|altavista|AppEngine-Google|archive.{0,30}\.org_bot|archiver|Ask Jeeves|[Bb]ai[Dd]u[Ss]pider(?:-[A-Za-z]{1,30})(?:-[A-Za-z]{1,30}|)|bingbot|BingPreview|blitzbot|BlogBridge|Bloglovin|BoardReader Blog Indexer|BoardReader Favicon Fetcher|boitho.com-dc|BotSeer|BUbiNG|\b\w{0,30}favicon\w{0,30}\b|\bYeti(?:-[a-z]{1,30}|)|Catchpoint(?: bot|)|[Cc]harlotte|Checklinks|clumboot|Comodo HTTP\(S\) Crawler|Comodo-Webinspector-Crawler|ConveraCrawler|CRAWL-E|CrawlConvera|Daumoa(?:-feedfetcher|)|Feed Seeker Bot|Feedbin|findlinks|Flamingo_SearchEngine|FollowSite Bot|furlbot|Genieo|gigabot|GomezAgent|gonzo1|(?:[a-zA-Z]{1,30}-|)Googlebot(?:-[a-zA-Z]{1,30}|)|Google SketchUp|grub-client|gsa-crawler|heritrix|HiddenMarket|holmes|HooWWWer|htdig|ia_archiver|ICC-Crawler|Icarus6j|ichiro(?:/mobile|)|IconSurf|IlTrovatore(?:-Setaccio|)|InfuzApp|Innovazion Crawler|InternetArchive|IP2[a-z]{1,30}Bot|jbot\b|KaloogaBot|Kraken|Kurzor|larbin|LEIA|LesnikBot|Linguee Bot|LinkAider|LinkedInBot|Lite Bot|Llaut|lycos|Mail\.RU_Bot|masscan|masidani_bot|Mediapartners-Google|Microsoft .{0,30} Bot|mogimogi|mozDex|MJ12bot|msnbot(?:-media {0,2}|)|msrbot|Mtps Feed Aggregation System|netresearch|Netvibes|NewsGator[^/]{0,30}|^NING|Nutch[^/]{0,30}|Nymesis|ObjectsSearch|OgScrper|Orbiter|OOZBOT|PagePeeker|PagesInventory|PaxleFramework|Peeplo Screenshot Bot|PlantyNet_WebRobot|Pompos|Qwantify|Read%20Later|Reaper|RedCarpet|Retreiver|Riddler|Rival IQ|scooter|Scrapy|Scrubby|searchsight|seekbot|semanticdiscovery|SemrushBot|Simpy|SimplePie|SEOstats|SimpleRSS|SiteCon|Slackbot-LinkExpanding|Slack-ImgProxy|Slurp|snappy|Speedy Spider|Squrl Java|Stringer|TheUsefulbot|ThumbShotsBot|Thumbshots\.ru|Tiny Tiny RSS|Twitterbot|WhatsApp|URL2PNG|Vagabondo|VoilaBot|^vortex|Votay bot|^voyager|WASALive.Bot|Web-sniffer|WebThumb|WeSEE:[A-z]{1,30}|WhatWeb|WIRE|WordPress|Wotbox|www\.almaden\.ibm\.com|Xenu(?:.s|) Link Sleuth|Xerka [A-z]{1,30}Bot|yacy(?:bot|)|YahooSeeker|Yahoo! Slurp|Yandex\w{1,30}|YodaoBot(?:-[A-z]{1,30}|)|YottaaMonitor|Yowedo|^Zao|^Zao-Crawler|ZeBot_www\.ze\.bz|ZooShot|ZyBorg|ArcGIS Hub Indexer)(?:[ /]v?(\d+)(?:\.(\d+)(?:\.(\d+)|)|)|)'
 
   # AWS S3 Clients
   # must come before "Bots General matcher" to catch "boto"/"boto3" before "bot"
   - regex: '\b(Boto3?|JetS3t|aws-(?:cli|sdk-(?:cpp|go|java|nodejs|ruby2?|dotnet-(?:\d{1,2}|core)))|s3fs)/(\d+)\.(\d+)(?:\.(\d+)|)'
 
+  # SAFE FME
+  - regex: '(FME)\/(\d+\.\d+)\.(\d+)\.(\d+)'
+
+  # QGIS
+  - regex: '(QGIS)\/(\d)\.?0?(\d{1,2})\.?0?(\d{1,2})'
+
+  # JOSM
+  - regex: '(JOSM)/(\d+)\.(\d+)'
+
+  # Tygron Platform
+  - regex: '(Tygron Platform) \((\d+)\.(\d+)\.(\d+(?:\.\d+| RC \d+\.\d+))'
+
   # Facebook
   # Must come before "Bots General matcher" to catch OrangeBotswana
   # Facebook Messenger must go before Facebook
@@ -415,6 +471,9 @@ user_agent_parsers:
   - regex: '(Rackspace Monitoring)/(\d+)\.(\d+)'
     family_replacement: 'RackspaceBot'
 
+  # PRTG Network Monitoring
+  - regex: '(PRTG Network Monitor)'
+
   # PyAMF
   - regex: '(PyAMF)/(\d+)\.(\d+)\.(\d+)'
 
@@ -848,7 +907,7 @@ user_agent_parsers:
     family_replacement: 'Python Requests'
 
   # headless user-agents
-  - regex: '\b(Windows-Update-Agent|Microsoft-CryptoAPI|SophosUpdateManager|SophosAgent|Debian APT-HTTP|Ubuntu APT-HTTP|libcurl-agent|libwww-perl|urlgrabber|curl|PycURL|Wget|wget2|aria2|Axel|OpenBSD ftp|lftp|jupdate|insomnia|fetch libfetch|akka-http|got)(?:[ /](\d+)(?:\.(\d+)|)(?:\.(\d+)|)|)'
+  - regex: '\b(Windows-Update-Agent|WindowsPowerShell|Microsoft-CryptoAPI|SophosUpdateManager|SophosAgent|Debian APT-HTTP|Ubuntu APT-HTTP|libcurl-agent|libwww-perl|urlgrabber|curl|PycURL|Wget|wget2|aria2|Axel|OpenBSD ftp|lftp|jupdate|insomnia|fetch libfetch|akka-http|got)(?:[ /](\d+)(?:\.(\d+)|)(?:\.(\d+)|)|)'
 
   # Asynchronous HTTP Client/Server for asyncio and Python (https://aiohttp.readthedocs.io/)
   - regex: '(Python/3\.\d{1,3} aiohttp)/(\d+)\.(\d+)\.(\d+)'
@@ -958,6 +1017,12 @@ os_parsers:
   - regex: '(CPU[ +]OS|iPhone[ +]OS|CPU[ +]iPhone)[ +]+(\d+)[_\.](\d+)(?:[_\.](\d+)|).*Outlook-iOS-Android'
     os_replacement: 'iOS'
 
+  # Special case for old ArcGIS Mobile products
+  - regex: 'ArcGIS\.?(iOS|Android)-\d+\.\d+(?:\.\d+|)(?:[^\/]+|)\/(\d+)(?:\.(\d+)(?:\.(\d+)|)|)'
+
+  # Special case for new ArcGIS Mobile products
+  - regex: 'ArcGISRuntime-(?:Android|iOS)\/\d+\.\d+(?:\.\d+|) \((Android|iOS) (\d+)(?:\.(\d+)(?:\.(\d+)|)|);'
+
   ##########
   # Android
   # can actually detect rooted android os. do we care?
@@ -1041,11 +1106,11 @@ os_parsers:
     os_replacement: 'Windows'
     os_v1_replacement: 'XP'
 
-  - regex: '(Windows NT 6\.1)'
+  - regex: '(Win(?:dows NT |32NT\/)6\.1)'
     os_replacement: 'Windows'
     os_v1_replacement: '7'
 
-  - regex: '(Windows NT 6\.0)'
+  - regex: '(Win(?:dows NT |32NT\/)6\.0)'
     os_replacement: 'Windows'
     os_v1_replacement: 'Vista'
 
@@ -1057,7 +1122,7 @@ os_parsers:
     os_replacement: 'Windows'
     os_v1_replacement: 'RT'
 
-  - regex: '(Windows NT 6\.2)'
+  - regex: '(Win(?:dows NT |32NT\/)6\.2)'
     os_replacement: 'Windows'
     os_v1_replacement: '8'
 
@@ -1066,12 +1131,12 @@ os_parsers:
     os_v1_replacement: 'RT 8'
     os_v2_replacement: '1'
 
-  - regex: '(Windows NT 6\.3)'
+  - regex: '(Win(?:dows NT |32NT\/)6\.3)'
     os_replacement: 'Windows'
     os_v1_replacement: '8'
     os_v2_replacement: '1'
 
-  - regex: '(Windows NT 6\.4)'
+  - regex: '(Win(?:dows NT |32NT\/)6\.4)'
     os_replacement: 'Windows'
     os_v1_replacement: '10'
 

tests/test_ua.yaml

@@ -8104,6 +8104,7 @@ test_cases:
     major: '8'
     minor: '2'
     patch: '2'
+
   - user_agent_string: 'Snapchat/10.29.1.0 (iPhone10,1; iOS 11.2; gzip)'
     family: 'Snapchat'
     major: '10'
@@ -8147,9 +8148,189 @@ test_cases:
     major:
     minor:
     patch:
-
+  
   - user_agent_string: 'Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; AspiegelBot)'
     family: 'AspiegelBot'
     major:
     minor:
     patch:
+
+  - user_agent_string: 'Mozilla/5.0 QGIS/31100'
+    family: 'QGIS'
+    major: '3'
+    minor: '11'
+    patch: '0'
+
+  - user_agent_string: 'Mozilla/5.0 QGIS/2.18.20'
+    family: 'QGIS'
+    major: '2'
+    minor: '18'
+    patch: '20'
+
+  - user_agent_string: 'JOSM/1.5 (15492 nl) Windows 7 64-Bit Java/1.8.0_221'
+    family: 'JOSM'
+    major: '1'
+    minor: '5'
+    patch:
+
+  - user_agent_string: 'FME/2018.7.34.18312  libcurl/7.57.0 WinSSL zlib/1.2.11 WinIDN libssh2/1.7.0'
+    family: 'FME'
+    major: '2018.7'
+    minor: '34'
+    patch: '18312'
+
+  - user_agent_string: 'GeoEvent Server 10.5.1'
+    family: 'GeoEvent Server'
+    major: '10'
+    minor: '5'
+    patch: '1'
+
+  - user_agent_string: 'ArcGIS Pro 2.4.2 (000000000) - ArcGIS Pro'
+    family: 'ArcGIS Pro'
+    major: '2'
+    minor: '4'
+    patch: '2'
+
+  - user_agent_string: 'ArcGIS Client Using WinInet'
+    family: 'ArcMap'
+    major:
+    minor:
+    patch:
+
+  - user_agent_string: 'ArcGISRuntime-Android/10.3.0 (Android 4.4; armeabi-v7a; SAMSUNG-GT-I9195) arcgis-collector/18.0.3 (00000000-0000-0000-0000-000000000000)'
+    family: 'Collector for ArcGIS'
+    major: '18'
+    minor: '0'
+    patch: '3'
+
+  - user_agent_string: 'ArcGISRuntime-iOS/10.3.0 (iOS 10.2.1; iPad5,4) arcgis-collector/19.0.2 (00000000-0000-0000-0000-000000000000)'
+    family: 'Collector for ArcGIS'
+    major: '19'
+    minor: '0'
+    patch: '2'
+
+  - user_agent_string: 'ArcGISRuntime-iOS/100.4 (iOS 11.4; iPad7,6) arcgis-aurora/18.1.0 (00000000-0000-0000-0000-000000000000)'
+    family: 'Collector for ArcGIS'
+    major: '18'
+    minor: '1'
+    patch: '0'
+
+  - user_agent_string: 'ArcGISRuntime-NET/10.3.0 (Windows 10.0.18362; x64; UAP; Windows.Desktop) arcgis-collector/18.0.2 (00000000-0000-0000-0000-000000000000)'
+    family: 'Collector for ArcGIS'
+    major: '18'
+    minor: '0'
+    patch: '2'
+
+  - user_agent_string: 'Collector-Android-10.3.7/ArcGIS.Android-10.2.5/7.0/SAMSUNG-SM-T815'
+    family: 'Collector for ArcGIS'
+    major: '10'
+    minor: '3'
+    patch: '7'
+
+  - user_agent_string: 'Collector-iOS-10.4.0:ArcGISiOS-10.2.4+Collector/13.1.2/iPad5,4'
+    family: 'Collector for ArcGIS'
+    major: '10'
+    minor: '4'
+    patch: '0'
+
+  - user_agent_string: 'Collector/6212016 CFNetwork/978.0.7 Darwin/18.7.0'
+    family: 'Collector for ArcGIS'
+    major: '6212016'
+    minor:
+    patch:
+
+  - user_agent_string: 'ArcGISRuntime-Android/100.2.2 (Android 6.0; armeabi-v7a; SAMSUNG-SM-T800) arcgis-explorer/18.1.0 (00000000-0000-0000-0000-000000000000)'
+    family: 'Explorer for ArcGIS'
+    major: '18'
+    minor: '1'
+    patch: '0'
+
+  - user_agent_string: 'ArcGISRuntime-iOS/100.1 (iPhone OS 9.3.5; iPad2,2) arcgis-explorer/17.1.2 (00000000-0000-0000-0000-000000000000)'
+    family: 'Explorer for ArcGIS'
+    major: '17'
+    minor: '1'
+    patch: '2'
+
+  - user_agent_string: 'Explorer/1544 CFNetwork/1107.1 Darwin/19.0.0'
+    family: 'Explorer for ArcGIS'
+    major: '1544'
+    minor:
+    patch:
+
+  - user_agent_string: 'Explorer-Android-10.2.10/ArcGIS.Android-10.2.8/5.1.1/SAMSUNG-SM-G361F'
+    family: 'Explorer for ArcGIS'
+    major: '10'
+    minor: '2'
+    patch: '10'
+
+  - user_agent_string: 'Explorer-iOS-10.2.10:ArcGISiOS-10.2.4+Collector/13.1.3/iPhone9,3'
+    family: 'Explorer for ArcGIS'
+    major: '10'
+    minor: '2'
+    patch: '10'
+
+  - user_agent_string: 'ArcGISRuntime-Android/100.1.1 (Android 7.0; arm64-v8a; SAMSUNG-SM-G930F) arcgis-workforce/17.0.1 (00000000-0000-0000-0000-000000000000)'
+    family: 'Workforce for ArcGIS'
+    major: '17'
+    minor: '0'
+    patch: '1'
+
+  - user_agent_string: 'Workforce-iOS-17.0.1:ArcGISiOS-100.0.0.1529+dev/13.2/iPad6,12'
+    family: 'Workforce for ArcGIS'
+    major: '17'
+    minor: '0'
+    patch: '1'
+
+  - user_agent_string: 'AR/10.2.6.1704 .Net/Desktop (Win32NT/6.1.7601.65536; Win64) arcgisearth/1.5'
+    family: 'ArcGIS Earth'
+    major: '1'
+    minor: '5'
+    patch:
+
+  - user_agent_string: 'ArcGISRuntime-iOS/100.6 (iOS 12.3.1; iPad6,3) com.esri.earth.phone/1.0.0'
+    family: 'ArcGIS Earth'
+    major: '1'
+    minor: '0'
+    patch: '0'
+
+  - user_agent_string: 'ArcGISiOS-10.2.4/12.1/iPad5,2'
+    family: 'ArcGIS Runtime SDK for iOS'
+    major: '10'
+    minor: '2'
+    patch: '4'
+
+  - user_agent_string: 'ArcGISRuntime-iOS/100.1.1 (iOS 12.4.1; iPhone10,4)'
+    family: 'ArcGIS Runtime SDK for iOS'
+    major: '100'
+    minor: '1'
+    patch: '1'
+
+  - user_agent_string: 'ArcGIS.Android-10.2.4/8.0.0/SAMSUNG-SM-A530F'
+    family: 'ArcGIS Runtime SDK for Android'
+    major: '10'
+    minor: '2'
+    patch: '4'
+
+  - user_agent_string: 'ArcGIS.Android.10.2'
+    family: 'ArcGIS Runtime SDK for Android'
+    major: '10'
+    minor: '2'
+    patch: 
+
+  - user_agent_string: 'ArcGISRuntime-Android/100.4 (Android 10.0; arm64-v8a; ONEPLUS-HD1903)'
+    family: 'ArcGIS Runtime SDK for Android'
+    major: '100'
+    minor: '4'
+    patch:
+
+  - user_agent_string: 'ArcGISRuntime-Qt/100.3 (Windows 10; x86_64; Qt 5.12.1; C++)'
+    family: 'ArcGIS Runtime SDK for Qt'
+    major: '100'
+    minor: '3'
+    patch:
+
+  - user_agent_string: 'ArcGISRuntime-NET/100.7 (Windows 10.0.18362; Win64; WOW64; UAP; Windows.Desktop)'
+    family: 'ArcGIS Runtime SDK for NET'
+    major: '100'
+    minor: '7'
+    patch: