Merge pull request #1471 from p-l-/passive-top-ua

Passive: add useragent top value
ivre · Jan 12, 2023 · 8db188d · 8db188d
2 parents 8596003 + 08de4c9
commit 8db188d
Show file tree

Hide file tree

Showing 5 changed files with 45 additions and 6 deletions.
diff --git a/ivre/db/mongo.py b/ivre/db/mongo.py
@@ -5163,6 +5163,14 @@ def _fixval(value):
                 "fingerprint": "infos.md5",
                 "key": "value",
             }.get(subfield, f"infos.{subfield}")
+        elif field == "useragent" or field.startswith("useragent:"):
+            if field == "useragent":
+                flt = self.flt_and(flt, self.searchuseragent())
+            else:
+                flt = self.flt_and(
+                    flt, self.searchuseragent(useragent=utils.str2regexp(field[10:]))
+                )
+            field = "value"
         pipeline = self._topvalues(
             field,
             flt=flt,

diff --git a/ivre/db/sql/__init__.py b/ivre/db/sql/__init__.py
@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 
 # This file is part of IVRE.
-# Copyright 2011 - 2022 Pierre LALET <pierre@droids-corp.org>
+# Copyright 2011 - 2023 Pierre LALET <pierre@droids-corp.org>
 #
 # IVRE is free software: you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by
@@ -3024,6 +3024,14 @@ def outputproc(val):
                 "fingerprint": self.tables.passive.moreinfo["md5"],
                 "key": self.tables.passive.value,
             }.get(subfield, self.tables.passive.moreinfo[subfield])
+        elif field == "useragent" or field.startswith("useragent:"):
+            if field == "useragent":
+                flt = self.flt_and(flt, self.searchuseragent())
+            else:
+                flt = self.flt_and(
+                    flt, self.searchuseragent(useragent=utils.str2regexp(field[10:]))
+                )
+            field = self.tables.passive.value
 
         if not isinstance(field, list):
             field = [field]

diff --git a/ivre/db/tiny.py b/ivre/db/tiny.py
@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 
 # This file is part of IVRE.
-# Copyright 2011 - 2022 Pierre LALET <pierre@droids-corp.org>
+# Copyright 2011 - 2023 Pierre LALET <pierre@droids-corp.org>
 #
 # IVRE is free software: you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by
@@ -3180,6 +3180,19 @@ def _extractor(flt, field):
             else:
                 fields = [field, "count"]
 
+        elif field == "useragent" or field.startswith("useragent:"):
+            if field == "useragent":
+                flt = self.flt_and(flt, self.searchuseragent())
+            else:
+                flt = self.flt_and(
+                    flt, self.searchuseragent(useragent=utils.str2regexp(field[10:]))
+                )
+            field = "value"
+            if distinct:
+                fields = [field]
+            else:
+                fields = [field, "count"]
+
         if distinct:
             return [
                 {"_id": _outputproc(val), "count": count}

diff --git a/tests/samples/results b/tests/samples/results
@@ -444,6 +444,14 @@ passive_top_sshkey_bits_cgi = [['RSA', 1024]]
 passive_top_sshkey_bits_cli = ['RSA / 1024']
 passive_top_sshkey_fingerprint = ['207ce596b04ecea4dbe4aa29e8909807']
 passive_top_sshkey_keytype = ['RSA']
+passive_top_useragent_distinct = ['Mozilla/5.0 (Windows NT 6.0; rv:34.0) Gecko/20100101 Firefox/34.0', 'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.6) Gecko/20040113', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5', 'Mozilla/5.0 (X11; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0', 'Syncer/4.80 (av_pro-1169;f)', 'Wget', 'curl/7.33.0', 'curl/7.34.0', 'curl/7.38.0', 'curl/7.74.0']
+passive_top_useragent_distinct_count = 1
+passive_top_useragent_Windows_distinct = ['Mozilla/5.0 (Windows NT 6.0; rv:34.0) Gecko/20100101 Firefox/34.0', 'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.6) Gecko/20040113', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5']
+passive_top_useragent_Windows_distinct_count = 1
+passive_top_useragent_not_distinct = ['curl/7.74.0']
+passive_top_useragent_not_distinct_count = 27
+passive_top_useragent_Windows_not_distinct = ['Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko']
+passive_top_useragent_Windows_not_distinct_count = 11
 passive_torcert_count = 10
 view_count_active = 926
 view_count_hassh = 123

diff --git a/tests/tests.py b/tests/tests.py
@@ -1,7 +1,7 @@
 #! /usr/bin/env python
 
 # This file is part of IVRE.
-# Copyright 2011 - 2022 Pierre LALET <pierre@droids-corp.org>
+# Copyright 2011 - 2023 Pierre LALET <pierre@droids-corp.org>
 #
 # IVRE is free software: you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by
@@ -491,11 +491,11 @@ def _check_top_value_cgi(self, name, field, count=10, webroute="", **kwargs):
 
     def check_passive_top_value_api(self, name, field, distinct):
         values = self._sort_top_values(
-            ivre.db.db.passive.topvalues(field=field, distinct=distinct, topnbr=10)
+            ivre.db.db.passive.topvalues(field=field, distinct=distinct, topnbr=12)
         )
         self.check_value(name, values, check=self.assertCountEqual)
         cur = iter(
-            ivre.db.db.passive.topvalues(field=field, distinct=distinct, topnbr=10)
+            ivre.db.db.passive.topvalues(field=field, distinct=distinct, topnbr=12)
         )
         values = next(cur)
         while values.get("_id") is None:
@@ -2507,13 +2507,15 @@ def test_40_passive(self):
                 "domains:2",
                 "domains:com",
                 "domains:com:2",
+                "useragent",
+                "useragent:/Windows/",
             ]:
                 if DATABASE == "sqlite" and field.startswith("domains"):
                     # BUG in sqlite backend: cannot use topvalues with
                     # JSON fields
                     continue
                 valname = "passive_top_%s_%sdistinct" % (
-                    field.replace(":", "_"),
+                    field.replace(":", "_").replace("/", ""),
                     "" if distinct else "not_",
                 )
                 self.check_passive_top_value_api(valname, field, distinct)