Rollback from using ppdeep to python-ssdeep (libfuzzy) (#868)

* Rollback from using ppdeep to python-ssdeep (libfuzzy) * Adapt documentation
CERT-Polska · Jul 31, 2023 · 18c1339 · 18c1339
1 parent a7dc1f4
commit 18c1339
Show file tree

Hide file tree

Showing 5 changed files with 16 additions and 10 deletions.
diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile
@@ -4,7 +4,7 @@ WORKDIR /app
 RUN python3 -m venv /app/venv
 RUN /app/venv/bin/pip --no-cache-dir install wheel
 
-RUN apk add --no-cache libffi libffi-dev py3-cffi build-base python3-dev automake m4 autoconf libtool gcc g++ musl-dev openssl-dev cargo postgresql-dev
+RUN apk add --no-cache libffi libffi-dev py3-cffi build-base python3-dev automake m4 autoconf libtool gcc g++ musl-dev openssl-dev cargo postgresql-dev libfuzzy2-dev
 
 COPY requirements.txt /app
 RUN /app/venv/bin/pip --no-cache-dir install -r /app/requirements.txt
@@ -18,7 +18,7 @@ FROM python:3.8-alpine
 
 LABEL maintainer="info@cert.pl"
 
-RUN apk add --no-cache postgresql-client postgresql-dev libmagic
+RUN apk add --no-cache postgresql-client postgresql-dev libmagic libfuzzy2-dev
 
 # Copy backend files
 COPY --from=build /app/venv /app/venv

diff --git a/docs/setup-and-configuration.rst b/docs/setup-and-configuration.rst
@@ -43,6 +43,7 @@ For production environments, you need to install:
 
 
 * **PostgreSQL database** (minimum supported version: 12, https://www.postgresql.org/download/linux/debian/)
+* libfuzzy2 for ssdeep evaluation
 
 Optionally you can install:
 

diff --git a/docs/whats-changed.rst b/docs/whats-changed.rst
@@ -7,6 +7,16 @@ have compatibility problems after minor mwdb-core upgrade.
 
 For upgrade instructions, see :ref:`Upgrade mwdb-core to latest version`.
 
+v2.10.1
+-------
+
+In v2.9.0 we switched from native ssdeep implementation to Python-based ppdeep library. Unfortunately, we have not taken
+into account the large impact on performance. This bugfix release goes one step backwards and requires **libfuzzy2**
+native library to be installed on server.
+
+Complete changelog can be found here: `v2.10.0 changelog <https://github.com/CERT-Polska/mwdb-core/releases/tag/v2.10.0>`_.
+
+
 v2.10.0
 -------
 

diff --git a/mwdb/core/util.py b/mwdb/core/util.py
@@ -16,7 +16,7 @@ def token_hex(nbytes=None):
 import boto3
 import botocore.client
 import magic
-import ppdeep
+import ssdeep
 from botocore.credentials import (
     ContainerProvider,
     InstanceMetadataFetcher,
@@ -106,12 +106,7 @@ def calc_magic(stream) -> str:
 
 
 def calc_ssdeep(stream):
-    stream.seek(0, os.SEEK_END)
-    file_size = stream.tell()
-
-    stream.seek(0, os.SEEK_SET)
-
-    return ppdeep._spamsum(stream, file_size)
+    return calc_hash(stream, ssdeep.Hash(), lambda h: h.digest())
 
 
 def calc_crc32(stream):

diff --git a/requirements.txt b/requirements.txt
@@ -7,7 +7,6 @@ Flask-Migrate==3.1.0
 Flask-RESTful==0.3.9
 SQLAlchemy==1.3.18
 marshmallow==3.7.1
-ppdeep==20200505
 psycopg2-binary==2.8.5
 requests==2.31.0
 apispec[yaml,validation]==3.3.1
@@ -29,3 +28,4 @@ Flask-Limiter==2.1.3
 python-dateutil==2.8.2
 pyzipper==0.3.5
 pycryptodomex==3.16.0
+ssdeep==3.4