From 41a58d7ee88feda6fa127ce69ac2e83c78a1192a Mon Sep 17 00:00:00 2001 From: Moon Patel Date: Mon, 15 Apr 2024 12:52:12 +0530 Subject: [PATCH] fixes #1698 - hfinger analyzer (#2241) * hfinger analyzer * dependency for hfinger analyzer * migrations for hfinger analyzer * overridden update method * default config fix * modified usage.md * fix * fix --- .../file_analyzers/hfinger.py | 57 +++++++ .../0078_analyzer_config_hfinger.py | 152 ++++++++++++++++++ .../0031_add_hfinger_analyzer_free_to_use.py | 34 ++++ docker/Dockerfile | 3 +- docs/source/Usage.md | 1 + requirements/project-requirements.txt | 1 + tests/api_app/test_api.py | 2 +- 7 files changed, 248 insertions(+), 2 deletions(-) create mode 100644 api_app/analyzers_manager/file_analyzers/hfinger.py create mode 100644 api_app/analyzers_manager/migrations/0078_analyzer_config_hfinger.py create mode 100644 api_app/playbooks_manager/migrations/0031_add_hfinger_analyzer_free_to_use.py diff --git a/api_app/analyzers_manager/file_analyzers/hfinger.py b/api_app/analyzers_manager/file_analyzers/hfinger.py new file mode 100644 index 000000000..2e0b678ee --- /dev/null +++ b/api_app/analyzers_manager/file_analyzers/hfinger.py @@ -0,0 +1,57 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from hfinger.analysis import hfinger_analyze + +from api_app.analyzers_manager.classes import FileAnalyzer +from tests.mock_utils import if_mock_connections, patch + + +class Hfinger(FileAnalyzer): + """ + Create fingerprints of malware HTTP + requests stored in pcap files. + """ + + fingerprint_report_mode: int = 2 + + def run(self): + return hfinger_analyze(self.filepath, self.fingerprint_report_mode) + + @classmethod + def update(cls) -> bool: + pass + + @classmethod + def _monkeypatch(cls): + patches = [ + if_mock_connections( + patch( + "hfinger.analysis.hfinger_analyze", + return_value=[ + { + "epoch_time": "1388111476.787707000", + "ip_src": "192.168.1.138", + "ip_dst": "173.194.115.80", + "port_src": "49209", + "port_dst": "80", + "fingerprint": "2.4|1|0.5||2.4|1.2|GE|1|ac,ac-la,us-ag,\ + ac-en,ho,co|ac:te-ht,ap-xh+xm,as-as/ac-la:75ef792f/\ + us-ag:ca0c4d71/ac-en:gz,de/co:Ke-Al|||", + }, + { + "epoch_time": "1388111477.142485000", + "ip_src": "192.168.1.138", + "ip_dst": "66.225.230.141", + "port_src": "49220", + "port_dst": "80", + "fingerprint": "1.5|3|1.0|html|||GE|1|ac,re,ac-la,us-ag,\ + ac-en,ho,co|ac:te-ht,ap-xh+xm,as-as/ac-la:75ef792f/\ + us-ag:ca0c4d71/ac-en:gz,de/co:Ke-Al|||", + }, + ], + ) + ) + ] + + return super()._monkeypatch(patches=patches) diff --git a/api_app/analyzers_manager/migrations/0078_analyzer_config_hfinger.py b/api_app/analyzers_manager/migrations/0078_analyzer_config_hfinger.py new file mode 100644 index 000000000..719a890b3 --- /dev/null +++ b/api_app/analyzers_manager/migrations/0078_analyzer_config_hfinger.py @@ -0,0 +1,152 @@ +from django.db import migrations +from django.db.models.fields.related_descriptors import ( + ForwardManyToOneDescriptor, + ForwardOneToOneDescriptor, + ManyToManyDescriptor, +) + +plugin = { + "python_module": { + "health_check_schedule": None, + "update_schedule": None, + "module": "hfinger.Hfinger", + "base_path": "api_app.analyzers_manager.file_analyzers", + }, + "name": "Hfinger", + "description": "create fingerprints of malware HTTPs requests using [Hfinger](https://github.com/CERT-Polska/hfinger)", + "disabled": False, + "soft_time_limit": 30, + "routing_key": "default", + "health_check_status": True, + "type": "file", + "docker_based": False, + "maximum_tlp": "RED", + "observable_supported": [], + "supported_filetypes": ["application/vnd.tcpdump.pcap"], + "run_hash": False, + "run_hash_type": "", + "not_supported_filetypes": [], + "model": "analyzers_manager.AnalyzerConfig", +} + +params = [ + { + "python_module": { + "module": "hfinger.Hfinger", + "base_path": "api_app.analyzers_manager.file_analyzers", + }, + "name": "fingerprint_report_mode", + "type": "int", + "description": "Fingerprint report mode. \r\n0 - similar number of collisions and fingerprints as mode 2, but using fewer features, \r\n1 - representation of all designed features, but a little more collisions than modes 0, 2, and 4, \r\n2 - optimal (the default mode), \r\n3 - the lowest number of generated fingerprints, but the highest number of collisions, \r\n4 - the highest fingerprint entropy, but slightly more fingerprints than modes 0-2", + "is_secret": False, + "required": False, + } +] + +values = [ + { + "parameter": { + "python_module": { + "module": "hfinger.Hfinger", + "base_path": "api_app.analyzers_manager.file_analyzers", + }, + "name": "fingerprint_report_mode", + "type": "int", + "description": "Fingerprint report mode. \r\n0 - similar number of collisions and fingerprints as mode 2, but using fewer features, \r\n1 - representation of all designed features, but a little more collisions than modes 0, 2, and 4, \r\n2 - optimal (the default mode), \r\n3 - the lowest number of generated fingerprints, but the highest number of collisions, \r\n4 - the highest fingerprint entropy, but slightly more fingerprints than modes 0-2", + "is_secret": False, + "required": False, + }, + "analyzer_config": "Hfinger", + "connector_config": None, + "visualizer_config": None, + "ingestor_config": None, + "pivot_config": None, + "for_organization": False, + "value": 2, + "updated_at": "2024-04-03T19:33:51.679066Z", + "owner": None, + } +] + + +def _get_real_obj(Model, field, value): + def _get_obj(Model, other_model, value): + if isinstance(value, dict): + real_vals = {} + for key, real_val in value.items(): + real_vals[key] = _get_real_obj(other_model, key, real_val) + value = other_model.objects.get_or_create(**real_vals)[0] + # it is just the primary key serialized + else: + if isinstance(value, int): + if Model.__name__ == "PluginConfig": + value = other_model.objects.get(name=plugin["name"]) + else: + value = other_model.objects.get(pk=value) + else: + value = other_model.objects.get(name=value) + return value + + if ( + type(getattr(Model, field)) + in [ForwardManyToOneDescriptor, ForwardOneToOneDescriptor] + and value + ): + other_model = getattr(Model, field).get_queryset().model + value = _get_obj(Model, other_model, value) + elif type(getattr(Model, field)) in [ManyToManyDescriptor] and value: + other_model = getattr(Model, field).rel.model + value = [_get_obj(Model, other_model, val) for val in value] + return value + + +def _create_object(Model, data): + mtm, no_mtm = {}, {} + for field, value in data.items(): + value = _get_real_obj(Model, field, value) + if type(getattr(Model, field)) is ManyToManyDescriptor: + mtm[field] = value + else: + no_mtm[field] = value + try: + o = Model.objects.get(**no_mtm) + except Model.DoesNotExist: + o = Model(**no_mtm) + o.full_clean() + o.save() + for field, value in mtm.items(): + attribute = getattr(o, field) + if value is not None: + attribute.set(value) + return False + return True + + +def migrate(apps, schema_editor): + Parameter = apps.get_model("api_app", "Parameter") + PluginConfig = apps.get_model("api_app", "PluginConfig") + python_path = plugin.pop("model") + Model = apps.get_model(*python_path.split(".")) + if not Model.objects.filter(name=plugin["name"]).exists(): + exists = _create_object(Model, plugin) + if not exists: + for param in params: + _create_object(Parameter, param) + for value in values: + _create_object(PluginConfig, value) + + +def reverse_migrate(apps, schema_editor): + python_path = plugin.pop("model") + Model = apps.get_model(*python_path.split(".")) + Model.objects.get(name=plugin["name"]).delete() + + +class Migration(migrations.Migration): + atomic = False + dependencies = [ + ("api_app", "0062_alter_parameter_python_module"), + ("analyzers_manager", "0077_analyzer_config_abusix"), + ] + + operations = [migrations.RunPython(migrate, reverse_migrate)] diff --git a/api_app/playbooks_manager/migrations/0031_add_hfinger_analyzer_free_to_use.py b/api_app/playbooks_manager/migrations/0031_add_hfinger_analyzer_free_to_use.py new file mode 100644 index 000000000..dfba5cbfe --- /dev/null +++ b/api_app/playbooks_manager/migrations/0031_add_hfinger_analyzer_free_to_use.py @@ -0,0 +1,34 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + + +from django.db import migrations + + +def migrate(apps, schema_editor): + playbook_config = apps.get_model("playbooks_manager", "PlaybookConfig") + AnalyzerConfig = apps.get_model("analyzers_manager", "AnalyzerConfig") + pc = playbook_config.objects.get(name="FREE_TO_USE_ANALYZERS") + pc.analyzers.add(AnalyzerConfig.objects.get(name="Hfinger").id) + pc.full_clean() + pc.save() + + +def reverse_migrate(apps, schema_editor): + playbook_config = apps.get_model("playbooks_manager", "PlaybookConfig") + AnalyzerConfig = apps.get_model("analyzers_manager", "AnalyzerConfig") + pc = playbook_config.objects.get(name="FREE_TO_USE_ANALYZERS") + pc.analyzers.remove(AnalyzerConfig.objects.get(name="Hfinger").id) + pc.full_clean() + pc.save() + + +class Migration(migrations.Migration): + dependencies = [ + ("playbooks_manager", "0030_add_tweetfeeds_to_free_analyzers"), + ("analyzers_manager", "0078_analyzer_config_hfinger"), + ] + + operations = [ + migrations.RunPython(migrate, reverse_migrate), + ] diff --git a/docker/Dockerfile b/docker/Dockerfile index 6b8910703..e373cdcc8 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -33,9 +33,10 @@ RUN mkdir -p ${LOG_PATH} \ # install required packages. some notes about:o # python3-psycopg2 is required to use PostgresSQL with Django # apache2-utils is required to execute htpasswd +# tshark is required for Hfinger file analyzer RUN apt-get update \ && apt-get install -y --no-install-recommends apt-utils libsasl2-dev libssl-dev netcat-traditional \ - vim libldap2-dev libfuzzy-dev net-tools python3-psycopg2 git apache2-utils \ + vim libldap2-dev libfuzzy-dev net-tools python3-psycopg2 git apache2-utils tshark \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* \ && pip3 install --no-cache-dir --upgrade pip diff --git a/docs/source/Usage.md b/docs/source/Usage.md index dc74f15c0..f77ec8a9d 100644 --- a/docs/source/Usage.md +++ b/docs/source/Usage.md @@ -82,6 +82,7 @@ The following is the list of the available analyzers you can run out-of-the-box. * `ELF_Info`: static ELF analysis with [pyelftools](https://github.com/eliben/pyelftools) and [telfhash](https://github.com/trendmicro/telfhash) * `File_Info`: static generic File analysis (hashes, magic and [exiftool](https://exiftool.org/)) * `Floss`: [Mandiant Floss](https://github.com/mandiant/flare-floss) Obfuscated String Solver in files +* `Hfinger`: create fingerprints of malware HTTPS requests using [Hfinger](https://github.com/CERT-Polska/hfinger) * `PE_Info`: static PE analysis with [pefile](https://github.com/mlodic/pefile) * `PEframe_Scan`: Perform static analysis on Portable Executable malware and malicious MS Office documents with [PeFrame](https://github.com/guelfoweb/peframe) * `PDF_Info`: static PDF analysis ([peepdf](https://github.com/jesparza/peepdf) + [pdfid](https://github.com/mlodic/pdfid)) diff --git a/requirements/project-requirements.txt b/requirements/project-requirements.txt index 9a9be3701..a0a8b4244 100644 --- a/requirements/project-requirements.txt +++ b/requirements/project-requirements.txt @@ -74,6 +74,7 @@ greynoise==2.1.0 XLMMacroDeobfuscator[secure]==0.2.3 thinkst-zippy==0.1.2 querycontacts==2.0.0 +hfinger==0.2.2 # this is required because XLMMacroDeobfuscator does not pin the following packages pyxlsb2==0.0.8 diff --git a/tests/api_app/test_api.py b/tests/api_app/test_api.py index 6e7208813..08e02cb54 100644 --- a/tests/api_app/test_api.py +++ b/tests/api_app/test_api.py @@ -129,7 +129,7 @@ def test_analyze_file__pcap(self): self.assertEqual(md5, job.md5) self.assertCountEqual( - ["Suricata", "YARAify_File_Scan"], + ["Suricata", "YARAify_File_Scan", "Hfinger"], list(job.analyzers_to_execute.all().values_list("name", flat=True)), )