Feature/level2 cleanup

.gitignore

@@ -3,3 +3,4 @@ tmp/
 *.pyc
 venv/
 __pycache__/
+.idea

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "data/yara-rules"]
+	path = data/yara-rules
+	url = https://github.com/Yara-Rules/rules.git

docker/Dockerfile

@@ -1,12 +1,56 @@
-FROM python:3.5
-LABEL maintainer="PolySwarm Developers <info@polyswarm.io>"
+FROM alpine:3.7
+LABEL maintainer "PolySwarm Developers <info@polyswarm.io>, original YARA-alpine Dockerfile taken from https://github.com/blacktop"
+ENV YARA_VERSION 3.7.1
+ENV YARA_PY_VERSION 3.7.0
 
-WORKDIR /usr/src/app
+## INSTALL YARA DEPS, BUILD YARA
+RUN apk add --no-cache openssl file jansson bison python python3 tini su-exec && python3 -m ensurepip &&\
+pip3 install --upgrade pip setuptools && \
+if [ ! -e /usr/bin/pip ]; then ln -s pip3 /usr/bin/pip ; fi && \
+if [[ ! -e /usr/bin/python ]]; then ln -sf /usr/bin/python3 /usr/bin/python; fi
+RUN apk add --no-cache -t .build-deps py-setuptools \
+                                openssl-dev \
+                                jansson-dev \
+                                python-dev \
+                                python3-dev \
+                                build-base \
+                                libc-dev \
+                                file-dev \
+                                automake \
+                                autoconf \
+                                libtool \
+                                flex \
+                                git \
+                                git \
+  && set -x \
+  && echo "Install Yara from source..." \
+  && cd /tmp/ \
+  && git clone --recursive --branch v$YARA_VERSION https://github.com/VirusTotal/yara.git \
+  && cd /tmp/yara \
+  && ./bootstrap.sh \
+  && sync \
+  && ./configure --with-crypto \
+                 --enable-magic \
+                 --enable-cuckoo \
+                 --enable-dotnet \
+  && make \
+  && make install \
+  && echo "Install yara-python..." \
+  && cd /tmp/ \
+  && git clone --recursive --branch v$YARA_PY_VERSION https://github.com/VirusTotal/yara-python \
+  && cd yara-python \
+  && python3 setup.py build --dynamic-linking \
+  && python3 setup.py install \
+  && echo "Make test_rule..." \
+  && mkdir /rules \
+  && echo "rule dummy { condition: true }" > /rules/test_rule \
+  && rm -rf /tmp/*
 
+## COPY IN MICROENGINE FILES
+WORKDIR /usr/src/app
 COPY requirements.txt ./
-RUN set -x && pip install --no-cache-dir -r requirements.txt
-
 COPY . .
-RUN set -x && pip install .
+RUN set -x && pip install --no-cache-dir -r requirements.txt
 
+## DONE
 CMD ["microengine"]

requirements.txt

@@ -2,5 +2,8 @@ clamd==1.0.2
 click==6.7
 aiodns==1.1.1
 aiohttp==2.3.1
+yara-python==3.7.0
 web3==4.4.1
 websockets==5.0.1
+git+https://github.com/polyswarm/malwarerepoclient.git#egg=malwarerepoclient
+.

scripts/wait_and_both.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+./scripts/wait_for_it.sh $POLYSWARMD_HOST:$POLYSWARMD_PORT -t 0
+microengine --keyfile docker/keyfile --password password --backend multi

scripts/wait_and_run.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 ./scripts/wait_for_it.sh $POLYSWARMD_HOST:$POLYSWARMD_PORT -t 0
-microengine 
+microengine $*

setup.py

@@ -1,9 +1,6 @@
 from setuptools import setup
 
 
-def parse_requirements():
-    with open('requirements.txt', 'r') as f:
-        return f.read().splitlines()
 
 
 setup(
@@ -14,7 +11,6 @@ def parse_requirements():
     author_email='info@polyswarm.io',
     url='https://github.com/polyswarm/microengine',
     license='MIT',
-    install_requires=parse_requirements(),
     include_package_data=True,
     packages=['microengine'],
     package_dir={

src/microengine/__main__.py

@@ -23,8 +23,8 @@ def main(polyswarmd_addr, keyfile, password, backend, testing):
         backend (str): Backend implementation to use
         testing (int): Mode to process N bounties then exit (optional)
     """
-    micro_engine_class = choose_backend(backend)
 
+    micro_engine_class = choose_backend(backend)
     micro_engine_class(polyswarmd_addr, keyfile, password).run(testing)
 
 if __name__ == '__main__':

src/microengine/backend.py

@@ -2,15 +2,17 @@
 
 from microengine.clamav import ClamavMicroengine
 from microengine.eicar import EicarMicroengine
+from microengine.multi import MultiMicroengine
 from microengine.scratch import ScratchMicroengine
+from microengine.yara import YaraMicroengine
 
 
 def choose_backend(backend):
     """Resolves microengine name string to implementation
 
     Args:
         backend (str): Name of the backend to load, either one of the predefined
-            implementations or the name of a module to load
+            implementations or the name of a module to load (module:ClassName syntax or default of module:Microengine)
     Returns:
         (Class): Microengine class of the selected implementation
     Raises:
@@ -24,10 +26,17 @@ def choose_backend(backend):
         micro_engine_class = EicarMicroengine  # EicarMicroengine(polyswarmd_addr, keyfile, password).run(testing)
     elif backend == 'clamav':
         micro_engine_class = ClamavMicroengine  # ClamavMicroengine(polyswarmd_addr, keyfile, password).run(testing)
+    elif backend == 'yara':
+        micro_engine_class = YaraMicroengine
+    elif backend == 'multi':
+        micro_engine_class = MultiMicroengine
     else:
-        micro_engine_module = importlib.import_module(backend)
-        micro_engine_class = micro_engine_module.Microengine
-        # todo make the class name customizable
+        import_l = backend.split(":")
+        micro_engine_module_s = import_l[0]
+
+        micro_engine_module = importlib.import_module(micro_engine_module_s)
+        micro_engine_class = micro_engine_module.Microengine if ":" not in backend else eval(
+            "micro_engine_module.{0}".format(import_l[1]))
 
     if micro_engine_class is None:
         raise Exception("No microengine backend found {0}".format(backend))

src/microengine/multi.py

@@ -0,0 +1,71 @@
+import yara
+from microengine import Microengine
+import tempfile
+import os
+import clamd
+from io import BytesIO
+
+# ClamAV config
+CLAMD_HOST = os.getenv('CLAMD_HOST', 'localhost')
+CLAMD_PORT = int(os.getenv('CLAMD_PORT', '3310'))
+CLAMD_TIMEOUT = 30.0
+
+# Yara rules import
+RULES_DIR = 'data/yara-rules/'
+
+
+class MultiMicroengine(Microengine):
+    """Microengine which matches yara rules and scans samples through clamd"""
+
+    def __init__(self, polyswarmd_addr, keyfile, password):
+        """Initialize a ClamAV/Yara microengine
+
+        Args:
+            polyswarmd_addr (str): Address of polyswarmd
+            keyfile (str): Path to private key file to use to sign transactions
+            password (str): Password to decrypt the encrypted private key
+        """
+        super().__init__(polyswarmd_addr, keyfile, password)
+        self.clamd = clamd.ClamdNetworkSocket(CLAMD_HOST, CLAMD_PORT,
+                                              CLAMD_TIMEOUT)
+        self.rules = yara.compile(RULES_DIR + "malware/MALW_Eicar")
+
+    async def scan(self, guid, content):
+        """Scan an artifact with ClamAV + YARA
+
+        Args:
+            guid (str): GUID of the bounty under analysis, use to track artifacts in the same bounty
+            content (bytes): Content of the artifact to be scan
+        Returns:
+            (bool, bool, str): Tuple of bit, verdict, metadata
+
+            bit (bool): Whether to include this artifact in the assertion or not
+            verdict (bool): Whether this artifact is malicious or not
+            metadata (str): Optional metadata about this artifact
+        """
+
+        yara_res = False
+        clam_res = False
+        yara_metadata = ''
+        clam_metadata = ''
+
+        # Yara rule matching
+        matches = self.rules.match(data=content)
+        if matches:
+            yara_res = True
+
+        # ClamAV scan
+        result = self.clamd.instream(BytesIO(content)).get('stream')
+        if len(result) >= 2 and result[0] == 'FOUND':
+            clam_res = True
+            clam_metadata = result[1]
+
+        # We assert on all artifacts
+        bit = True
+
+        # If either finds a match, trust it and send it along
+        # If not, assert it is benign
+        verdict = yara_res or clam_res
+        metadata = ' '.join([yara_metadata, clam_metadata]).strip()
+
+        return bit, verdict, metadata

src/microengine/yara.py

@@ -0,0 +1,40 @@
+import yara
+from microengine import Microengine
+import tempfile
+import os
+
+RULES_DIR = 'data/yara-rules/'
+
+
+class YaraMicroengine(Microengine):
+    """Microengine which matches samples against yara rules"""
+
+    def __init__(self, polyswarmd_addr, keyfile, password):
+        """Initialize a Yara microengine
+
+        Args:
+            polyswarmd_addr (str): Address of polyswarmd
+            keyfile (str): Path to private key file to use to sign transactions
+            password (str): Password to decrypt the encrypted private key
+        """
+        super().__init__(polyswarmd_addr, keyfile, password)
+        self.rules = yara.compile(RULES_DIR + "malware/MALW_Eicar")
+
+    async def scan(self, guid, content):
+        """Scan an artifact with YARA
+
+        Args:
+            guid (str): GUID of the bounty under analysis, use to track artifacts in the same bounty
+            content (bytes): Content of the artifact to be scan
+        Returns:
+            (bool, bool, str): Tuple of bit, verdict, metadata
+
+            bit (bool): Whether to include this artifact in the assertion or not
+            verdict (bool): Whether this artifact is malicious or not
+            metadata (str): Optional metadata about this artifact
+        """
+        matches = self.rules.match(data=content)
+        if matches:
+            return True, True, ''
+
+        return True, False, ''