fixes #1698 - hfinger analyzer (#2241)

* hfinger analyzer

* dependency for hfinger analyzer

* migrations for hfinger analyzer

* overridden update method

* default config fix

* modified usage.md

* fix

* fix

api_app/analyzers_manager/file_analyzers/hfinger.py

@@ -0,0 +1,57 @@
+# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl
+# See the file 'LICENSE' for copying permission.
+
+from hfinger.analysis import hfinger_analyze
+
+from api_app.analyzers_manager.classes import FileAnalyzer
+from tests.mock_utils import if_mock_connections, patch
+
+
+class Hfinger(FileAnalyzer):
+    """
+    Create fingerprints of malware HTTP
+    requests stored in pcap files.
+    """
+
+    fingerprint_report_mode: int = 2
+
+    def run(self):
+        return hfinger_analyze(self.filepath, self.fingerprint_report_mode)
+
+    @classmethod
+    def update(cls) -> bool:
+        pass
+
+    @classmethod
+    def _monkeypatch(cls):
+        patches = [
+            if_mock_connections(
+                patch(
+                    "hfinger.analysis.hfinger_analyze",
+                    return_value=[
+                        {
+                            "epoch_time": "1388111476.787707000",
+                            "ip_src": "192.168.1.138",
+                            "ip_dst": "173.194.115.80",
+                            "port_src": "49209",
+                            "port_dst": "80",
+                            "fingerprint": "2.4|1|0.5||2.4|1.2|GE|1|ac,ac-la,us-ag,\
+                                ac-en,ho,co|ac:te-ht,ap-xh+xm,as-as/ac-la:75ef792f/\
+                                us-ag:ca0c4d71/ac-en:gz,de/co:Ke-Al|||",
+                        },
+                        {
+                            "epoch_time": "1388111477.142485000",
+                            "ip_src": "192.168.1.138",
+                            "ip_dst": "66.225.230.141",
+                            "port_src": "49220",
+                            "port_dst": "80",
+                            "fingerprint": "1.5|3|1.0|html|||GE|1|ac,re,ac-la,us-ag,\
+                                ac-en,ho,co|ac:te-ht,ap-xh+xm,as-as/ac-la:75ef792f/\
+                                us-ag:ca0c4d71/ac-en:gz,de/co:Ke-Al|||",
+                        },
+                    ],
+                )
+            )
+        ]
+
+        return super()._monkeypatch(patches=patches)

api_app/analyzers_manager/migrations/0078_analyzer_config_hfinger.py

@@ -0,0 +1,152 @@
+from django.db import migrations
+from django.db.models.fields.related_descriptors import (
+    ForwardManyToOneDescriptor,
+    ForwardOneToOneDescriptor,
+    ManyToManyDescriptor,
+)
+
+plugin = {
+    "python_module": {
+        "health_check_schedule": None,
+        "update_schedule": None,
+        "module": "hfinger.Hfinger",
+        "base_path": "api_app.analyzers_manager.file_analyzers",
+    },
+    "name": "Hfinger",
+    "description": "create fingerprints of malware HTTPs requests using [Hfinger](https://github.com/CERT-Polska/hfinger)",
+    "disabled": False,
+    "soft_time_limit": 30,
+    "routing_key": "default",
+    "health_check_status": True,
+    "type": "file",
+    "docker_based": False,
+    "maximum_tlp": "RED",
+    "observable_supported": [],
+    "supported_filetypes": ["application/vnd.tcpdump.pcap"],
+    "run_hash": False,
+    "run_hash_type": "",
+    "not_supported_filetypes": [],
+    "model": "analyzers_manager.AnalyzerConfig",
+}
+
+params = [
+    {
+        "python_module": {
+            "module": "hfinger.Hfinger",
+            "base_path": "api_app.analyzers_manager.file_analyzers",
+        },
+        "name": "fingerprint_report_mode",
+        "type": "int",
+        "description": "Fingerprint report mode. \r\n0 - similar number of collisions and fingerprints as mode 2, but using fewer features, \r\n1 - representation of all designed features, but a little more collisions than modes 0, 2, and 4, \r\n2 - optimal (the default mode), \r\n3 - the lowest number of generated fingerprints, but the highest number of collisions, \r\n4 - the highest fingerprint entropy, but slightly more fingerprints than modes 0-2",
+        "is_secret": False,
+        "required": False,
+    }
+]
+
+values = [
+    {
+        "parameter": {
+            "python_module": {
+                "module": "hfinger.Hfinger",
+                "base_path": "api_app.analyzers_manager.file_analyzers",
+            },
+            "name": "fingerprint_report_mode",
+            "type": "int",
+            "description": "Fingerprint report mode. \r\n0 - similar number of collisions and fingerprints as mode 2, but using fewer features, \r\n1 - representation of all designed features, but a little more collisions than modes 0, 2, and 4, \r\n2 - optimal (the default mode), \r\n3 - the lowest number of generated fingerprints, but the highest number of collisions, \r\n4 - the highest fingerprint entropy, but slightly more fingerprints than modes 0-2",
+            "is_secret": False,
+            "required": False,
+        },
+        "analyzer_config": "Hfinger",
+        "connector_config": None,
+        "visualizer_config": None,
+        "ingestor_config": None,
+        "pivot_config": None,
+        "for_organization": False,
+        "value": 2,
+        "updated_at": "2024-04-03T19:33:51.679066Z",
+        "owner": None,
+    }
+]
+
+
+def _get_real_obj(Model, field, value):
+    def _get_obj(Model, other_model, value):
+        if isinstance(value, dict):
+            real_vals = {}
+            for key, real_val in value.items():
+                real_vals[key] = _get_real_obj(other_model, key, real_val)
+            value = other_model.objects.get_or_create(**real_vals)[0]
+        # it is just the primary key serialized
+        else:
+            if isinstance(value, int):
+                if Model.__name__ == "PluginConfig":
+                    value = other_model.objects.get(name=plugin["name"])
+                else:
+                    value = other_model.objects.get(pk=value)
+            else:
+                value = other_model.objects.get(name=value)
+        return value
+
+    if (
+        type(getattr(Model, field))
+        in [ForwardManyToOneDescriptor, ForwardOneToOneDescriptor]
+        and value
+    ):
+        other_model = getattr(Model, field).get_queryset().model
+        value = _get_obj(Model, other_model, value)
+    elif type(getattr(Model, field)) in [ManyToManyDescriptor] and value:
+        other_model = getattr(Model, field).rel.model
+        value = [_get_obj(Model, other_model, val) for val in value]
+    return value
+
+
+def _create_object(Model, data):
+    mtm, no_mtm = {}, {}
+    for field, value in data.items():
+        value = _get_real_obj(Model, field, value)
+        if type(getattr(Model, field)) is ManyToManyDescriptor:
+            mtm[field] = value
+        else:
+            no_mtm[field] = value
+    try:
+        o = Model.objects.get(**no_mtm)
+    except Model.DoesNotExist:
+        o = Model(**no_mtm)
+        o.full_clean()
+        o.save()
+        for field, value in mtm.items():
+            attribute = getattr(o, field)
+            if value is not None:
+                attribute.set(value)
+        return False
+    return True
+
+
+def migrate(apps, schema_editor):
+    Parameter = apps.get_model("api_app", "Parameter")
+    PluginConfig = apps.get_model("api_app", "PluginConfig")
+    python_path = plugin.pop("model")
+    Model = apps.get_model(*python_path.split("."))
+    if not Model.objects.filter(name=plugin["name"]).exists():
+        exists = _create_object(Model, plugin)
+        if not exists:
+            for param in params:
+                _create_object(Parameter, param)
+            for value in values:
+                _create_object(PluginConfig, value)
+
+
+def reverse_migrate(apps, schema_editor):
+    python_path = plugin.pop("model")
+    Model = apps.get_model(*python_path.split("."))
+    Model.objects.get(name=plugin["name"]).delete()
+
+
+class Migration(migrations.Migration):
+    atomic = False
+    dependencies = [
+        ("api_app", "0062_alter_parameter_python_module"),
+        ("analyzers_manager", "0077_analyzer_config_abusix"),
+    ]
+
+    operations = [migrations.RunPython(migrate, reverse_migrate)]

api_app/playbooks_manager/migrations/0031_add_hfinger_analyzer_free_to_use.py

@@ -0,0 +1,34 @@
+# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl
+# See the file 'LICENSE' for copying permission.
+
+
+from django.db import migrations
+
+
+def migrate(apps, schema_editor):
+    playbook_config = apps.get_model("playbooks_manager", "PlaybookConfig")
+    AnalyzerConfig = apps.get_model("analyzers_manager", "AnalyzerConfig")
+    pc = playbook_config.objects.get(name="FREE_TO_USE_ANALYZERS")
+    pc.analyzers.add(AnalyzerConfig.objects.get(name="Hfinger").id)
+    pc.full_clean()
+    pc.save()
+
+
+def reverse_migrate(apps, schema_editor):
+    playbook_config = apps.get_model("playbooks_manager", "PlaybookConfig")
+    AnalyzerConfig = apps.get_model("analyzers_manager", "AnalyzerConfig")
+    pc = playbook_config.objects.get(name="FREE_TO_USE_ANALYZERS")
+    pc.analyzers.remove(AnalyzerConfig.objects.get(name="Hfinger").id)
+    pc.full_clean()
+    pc.save()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("playbooks_manager", "0030_add_tweetfeeds_to_free_analyzers"),
+        ("analyzers_manager", "0078_analyzer_config_hfinger"),
+    ]
+
+    operations = [
+        migrations.RunPython(migrate, reverse_migrate),
+    ]

docker/Dockerfile

@@ -33,9 +33,10 @@ RUN mkdir -p ${LOG_PATH} \
 # install required packages. some notes about:o
 # python3-psycopg2 is required to use PostgresSQL with Django
 # apache2-utils is required to execute htpasswd
+# tshark is required for Hfinger file analyzer
 RUN apt-get update \
     && apt-get install -y --no-install-recommends apt-utils libsasl2-dev libssl-dev netcat-traditional \
-    vim libldap2-dev libfuzzy-dev net-tools python3-psycopg2 git apache2-utils \
+    vim libldap2-dev libfuzzy-dev net-tools python3-psycopg2 git apache2-utils tshark \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/* \
     && pip3 install --no-cache-dir --upgrade pip

docs/source/Usage.md

@@ -82,6 +82,7 @@ The following is the list of the available analyzers you can run out-of-the-box.
 * `ELF_Info`: static ELF analysis with [pyelftools](https://github.com/eliben/pyelftools) and [telfhash](https://github.com/trendmicro/telfhash)
 * `File_Info`: static generic File analysis (hashes, magic and [exiftool](https://exiftool.org/))
 * `Floss`: [Mandiant Floss](https://github.com/mandiant/flare-floss) Obfuscated String Solver in files
+* `Hfinger`: create fingerprints of malware HTTPS requests using [Hfinger](https://github.com/CERT-Polska/hfinger)
 * `PE_Info`: static PE analysis with [pefile](https://github.com/mlodic/pefile)
 * `PEframe_Scan`: Perform static analysis on Portable Executable malware and malicious MS Office documents with [PeFrame](https://github.com/guelfoweb/peframe)
 * `PDF_Info`: static PDF analysis ([peepdf](https://github.com/jesparza/peepdf) + [pdfid](https://github.com/mlodic/pdfid))

requirements/project-requirements.txt

@@ -74,6 +74,7 @@ greynoise==2.1.0
 XLMMacroDeobfuscator[secure]==0.2.3
 thinkst-zippy==0.1.2
 querycontacts==2.0.0
+hfinger==0.2.2
 
 # this is required because XLMMacroDeobfuscator does not pin the following packages
 pyxlsb2==0.0.8

tests/api_app/test_api.py

@@ -129,7 +129,7 @@ def test_analyze_file__pcap(self):
         self.assertEqual(md5, job.md5)
 
         self.assertCountEqual(
-            ["Suricata", "YARAify_File_Scan"],
+            ["Suricata", "YARAify_File_Scan", "Hfinger"],
             list(job.analyzers_to_execute.all().values_list("name", flat=True)),
         )