integrated boxjs for .js scans

.env_template

@@ -10,15 +10,15 @@
 COMPOSE_FILE=docker-compose.yml
 
 # To run all additional integrations in production
-#COMPOSE_FILE=docker-compose.yml:./integrations/docker-compose.peframe.yml:./integrations/docker-compose.thug.yml:./integrations/docker-compose.capa.yml
+#COMPOSE_FILE=docker-compose.yml:./integrations/docker-compose.peframe.yml:./integrations/docker-compose.thug.yml:./integrations/docker-compose.capa.yml:./integrations/docker-compose.boxjs.yml
 
 ###### For Tests or local development ######
 
 #COMPOSE_FILE=docker-compose-for-tests.yml
 
 # To run all additional integrations in development
-#COMPOSE_FILE=docker-compose-for-tests.yml:./integrations/docker-compose-for-tests.peframe.yml:./integrations/docker-compose-for-tests.thug.yml:./integrations/docker-compose-for-tests.capa.yml
+#COMPOSE_FILE=docker-compose-for-tests.yml:./integrations/docker-compose-for-tests.peframe.yml:./integrations/docker-compose-for-tests.thug.yml:./integrations/docker-compose-for-tests.capa.yml:./integrations/docker-compose-for-tests.boxjs.yml
 
 ###### For travis ######
 
-#COMPOSE_FILE=docker-compose-for-tests.yml:./integrations/docker-compose-for-tests.peframe.yml:./integrations/docker-compose-for-tests.thug.yml:./integrations/docker-compose-for-tests.capa.yml
+#COMPOSE_FILE=docker-compose-for-tests.yml:./integrations/docker-compose-for-tests.peframe.yml:./integrations/docker-compose-for-tests.thug.yml:./integrations/docker-compose-for-tests.capa.yml:./integrations/docker-compose-for-tests.boxjs.yml

api_app/script_analyzers/file_analyzers/boxjs_scan.py

@@ -0,0 +1,52 @@
+import requests
+import logging
+
+from api_app.helpers import get_binary
+from api_app.script_analyzers.classes import FileAnalyzer, DockerBasedAnalyzer
+from api_app.exceptions import AnalyzerConfigurationException
+
+logger = logging.getLogger(__name__)
+
+
+class BoxJS(FileAnalyzer, DockerBasedAnalyzer):
+    name: str = "box-js"
+    base_url: str = "http://boxjs:4003"
+    url: str = f"{base_url}/boxjs"
+    # http request polling max number of tries
+    max_tries: int = 20
+    # interval between http request polling (in secs)
+    poll_distance: int = 10
+
+    def run(self):
+        # construct a valid filename into which thug will save the result
+        fname = str(self.filename).replace("/", "_").replace(" ", "_")
+        # get the file to send
+        binary = get_binary(self.job_id)
+        # construct arguments, For example this corresponds to,
+        # box-js sample.js --output-dir=result --timeout 200 --no-kill --no-shell-error
+        args = [
+            f"@{fname}",
+            "--output-dir=/tmp/boxjs",
+            "--timeout 200",
+            "--no-kill",
+            "--no-shell-error",
+            "--no-echo",
+        ]
+
+        # step #1: request new analysis
+        logger.debug(f"Making request with arguments: {args} <- {self.__repr__()}")
+        try:
+            resp1 = requests.post(
+                self.url, files={fname: binary}, data={"args": args,},
+            )
+        except requests.exceptions.ConnectionError:
+            raise AnalyzerConfigurationException(
+                f"{self.name} docker container is not running."
+            )
+
+        # step #2: raise AnalyzerRunException in case of error
+        assert self._raise_in_case_bad_request(self.name, resp1)
+
+        # step #3: if no error, continue try to fetch result
+        key = resp1.json().get("key", None)
+        return self._get_result_from_a_dir(key, fname)

api_app/script_analyzers/file_analyzers/peframe.py

@@ -9,12 +9,10 @@
 class PEframe(FileAnalyzer, DockerBasedAnalyzer):
     name: str = "PEframe"
     url: str = "http://peframe:4000/peframe"
-
-    def set_config(self, additional_config_params):
-        # http request polling max number of tries
-        self.max_tries: int = additional_config_params.get("max_tries", 15)
-        # interval between http request polling
-        self.poll_distance: int = additional_config_params.get("poll_distance", 5)
+    # http request polling max number of tries
+    max_tries: int = 25
+    # interval between http request polling
+    poll_distance: int = 5
 
     def run(self):
         # get binary

configuration/analyzer_config.json

@@ -720,12 +720,13 @@
     "PEframe_Scan": {
         "type": "file",
         "python_module": "peframe_run",
-        "description": "Perform static analysis on Portable Executable malware and malicious MS Office documents",
-        "requires_configuration": true,
-        "additional_config_params": {
-            "max_tries": 15,
-            "poll_distance": 5
-        }
+        "description": "Perform static analysis on Portable Executable malware and malicious MS Office documents"
+    },
+    "BoxJS_Scan_JavaScript": {
+        "type": "file",
+        "supported_filetypes": ["application/x-javascript", "application/javascript", "text/javascript"],
+        "python_module": "boxjs_run",
+        "description": "A tool for studying JavaScript malware"
     },
     "Capa_Info": {
         "type": "file",

docs/source/Installation.md

@@ -146,6 +146,11 @@ table, th, td {
     <td>4002</td>
     <td><code>Capa_Info</code></td>
   </tr>
+  <tr>
+    <td>Box-JS</td>
+    <td>4003</td>
+    <td><code>BoxJS_Scan_JavaScript</code></td>
+  </tr>
 </table>
 
 In the project, you can find template files named `.env_template` and `.env_file_integrations_template`.

docs/source/Usage.md

@@ -59,7 +59,8 @@ The following is the list of the available analyzers you can run out-of-the-box:
 * PEframe_Scan: Perform static analysis on Portable Executable malware and malicious MS Office documents.
 * Cymru_Hash_Registry_Get_File: Check if a particular file is known to be malware by Team Cymru
 * Thug_HTML_Info_*: Perform hybrid dynamic/static analysis on a HTML file using [Thug low-interaction honeyclient](https://thug-honeyclient.readthedocs.io/)
-* Capa_Info: [Capa](https://github.com/fireeye/capa) detects capabilities in executable files
+* `Capa_Info`: [Capa](https://github.com/fireeye/capa) detects capabilities in executable files
+* `BoxJS_Scan_Javascript`: [Box-JS](https://github.com/CapacitorSet/box-js) is a tool for studying JavaScript malware. 
 
 #### Observable analyzers (ip, domain, url, hash)
 * VirusTotal_v3_Get_Observable: search an observable in the VirusTotal DB

integrations/box-js/Dockerfile

@@ -0,0 +1,27 @@
+FROM python:3.8.4-alpine3.12
+
+ENV PROJECT_PATH /opt/deploy
+ENV LOG_PATH /var/log/intel_owl/box-js
+
+# update and install packages
+RUN apk update && apk add --no-cache git nodejs npm file gcc m4
+
+# Add a new low-privileged user
+RUN adduser -DH --shell /sbin/login boxjs-user
+
+# Install Box-js
+RUN apk add --no-cache -U --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing aufs-util \
+    && npm install box-js --global --production \
+    && mkdir -p /tmp/boxjs \
+    && chown -R boxjs-user:boxjs-user /tmp/boxjs
+
+# Build Flask REST API
+WORKDIR ${PROJECT_PATH}/boxjs-flask
+COPY entrypoint.sh app.py requirements.txt ./
+RUN pip3 install -r requirements.txt --no-cache-dir \
+    && chown -R boxjs-user:boxjs-user . \
+    && chmod +x entrypoint.sh
+
+# Serve Flask application using gunicorn
+EXPOSE 4003
+ENTRYPOINT ["./entrypoint.sh"]

integrations/box-js/app.py

@@ -0,0 +1,81 @@
+# system imports
+import os
+import logging
+import json
+import shutil
+import secrets
+
+# web imports
+from flask import Flask, jsonify, make_response, safe_join, request
+from flask_executor import Executor
+from flask_shell2http import Shell2HTTP
+
+# Logging configuration
+# get flask-shell2http logger instance
+logger = logging.getLogger("flask_shell2http")
+# logger config
+formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+log_level = os.getenv("LOG_LEVEL", logging.INFO)
+log_path = os.getenv("LOG_PATH", "/var/log/intel_owl/box-js")
+# create new file handlers, files are created if doesn't already exists
+fh = logging.FileHandler(f"{log_path}/box-js.log")
+fh.setFormatter(formatter)
+fh.setLevel(log_level)
+fh_err = logging.FileHandler(f"{log_path}/box-js_errors.log")
+fh_err.setFormatter(formatter)
+fh_err.setLevel(logging.ERROR)
+# add the handlers to the logger
+logger.addHandler(fh)
+logger.addHandler(fh_err)
+logger.setLevel(log_level)
+
+# Globals
+app = Flask(__name__)
+app.config["SECRET_KEY"] = secrets.token_hex(16)
+executor = Executor(app)
+shell2http = Shell2HTTP(app, executor)
+
+# with this, we can make http calls to the endpoint: /boxjs
+shell2http.register_command(endpoint="boxjs", command_name="box-js")
+
+
+@app.route("/get-result")
+def get_result():
+    # user provides us with dir_name i.e {filename}.results
+    fname = request.args.get("name", None)
+    try:
+        if not fname:
+            raise Exception("No name in GET request's query params.")
+        dir_loc = safe_join("/tmp/boxjs", fname + ".results")
+        result = read_files_and_make_result(dir_loc)
+
+        return make_response(jsonify(result), 200)
+    except Exception as e:
+        return make_response(jsonify(error=str(e)), 400)
+
+
+def read_files_and_make_result(dir_loc):
+    result = {}
+    files_to_read = [
+        "IOC.json",
+        "snippets.json",
+        "resources.json",
+        "analysis.log",
+        "urls.json",
+        "active_urls.json",
+    ]
+    # Read output from files one by one
+    for fname in files_to_read:
+        try:
+            with open(safe_join(dir_loc, fname)) as fp:
+                if fname.endswith(".json"):
+                    result[fname] = json.load(fp)
+                else:
+                    result[fname] = fp.readlines()
+        except FileNotFoundError:
+            result[fname] = f"FileNotFoundError: {fname}"
+
+    # Remove the directory
+    shutil.rmtree(dir_loc, ignore_errors=True)
+
+    return result

integrations/box-js/entrypoint.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+touch ${LOG_PATH}/gunicorn_access.log ${LOG_PATH}/gunicorn_errors.log
+chown -R boxjs-user:boxjs-user ${LOG_PATH}
+su boxjs-user -s /bin/sh
+exec gunicorn 'app:app' \
+    --bind '0.0.0.0:4003' \
+    --user boxjs-user \
+    --log-level ${LOG_LEVEL} \
+    --access-logfile ${LOG_PATH}/gunicorn_access.log \
+    --error-logfile ${LOG_PATH}/gunicorn_errors.log

integrations/box-js/requirements.txt

@@ -0,0 +1,9 @@
+click==7.1.1
+Flask==1.1.2
+Flask-Executor==0.9.3
+Flask-Shell2HTTP==1.2.0
+itsdangerous==1.1.0
+Jinja2==2.11.2
+MarkupSafe==1.1.1
+Werkzeug==1.0.1
+gunicorn==20.0.4

integrations/docker-compose-for-tests.boxjs.yml

@@ -0,0 +1,21 @@
+# IMPORTANT: The version must match the version of docker-compose.yml
+---
+version: '3'
+
+# All additional integrations should be added following this format only.
+
+services:
+  boxjs:
+    build:
+      context: ./integrations/box-js
+      dockerfile: Dockerfile
+    container_name: intelowl_boxjs
+    restart: unless-stopped
+    expose:
+      - "4003"
+    env_file:
+      - env_file_integrations
+    volumes:
+      - generic_logs:/var/log/intel_owl
+    depends_on:
+      - uwsgi

integrations/docker-compose.boxjs.yml

@@ -0,0 +1,19 @@
+# IMPORTANT: The version must match the version of docker-compose.yml
+---
+version: '3'
+
+# All additional integrations should be added following this format only.
+
+services:
+  boxjs:
+    image: intelowlproject/intelowl_boxjs
+    container_name: intelowl_boxjs
+    restart: unless-stopped
+    expose:
+      - "4003"
+    env_file:
+      - env_file_integrations
+    volumes:
+      - generic_logs:/var/log/intel_owl
+    depends_on:
+      - uwsgi

integrations/thug/Dockerfile

@@ -2,7 +2,6 @@ FROM ubuntu:18.04
 
 # Step 1: Build and install thug
 
-USER root
 RUN apt-get update \
     && apt-get install -y --no-install-recommends \
       build-essential sudo python python3 python3-dev python3-setuptools \

intel_owl/tasks.py

@@ -17,6 +17,7 @@
     peframe,
     thug_file,
     capa_info,
+    boxjs_scan,
 )
 from api_app.script_analyzers.observable_analyzers import (
     abuseipdb,
@@ -747,3 +748,10 @@ def thug_url_run(
         observable_classification,
         additional_config_params,
     ).start()
+
+
+@shared_task(soft_time_limit=400)
+def boxjs_run(analyzer_name, job_id, filepath, filename, md5, additional_config_params):
+    boxjs_scan.BoxJS(
+        analyzer_name, job_id, filepath, filename, md5, additional_config_params
+    ).start()

tests/test_files.py

@@ -22,6 +22,7 @@
     peframe,
     thug_file,
     capa_info,
+    boxjs_scan,
 )
 from api_app.script_analyzers.observable_analyzers import vt3_get
 
@@ -361,6 +362,35 @@ def test_thug_html(self, mock_get=None, mock_post=None):
         self.assertEqual(report.get("success", False), True)
 
 
+class FileAnalyzersJSTests(TestCase):
+    def setUp(self):
+        params = {
+            "source": "test",
+            "is_sample": True,
+            "file_mimetype": "application/javascript",
+            "force_privacy": False,
+            "analyzers_requested": ["test"],
+        }
+        filename = "file.jse"
+        test_job = _generate_test_job_with_file(params, filename)
+        self.job_id = test_job.id
+        self.filepath, self.filename = general.get_filepath_filename(self.job_id)
+        self.md5 = test_job.md5
+
+    @mock_connections(patch("requests.get", side_effect=mocked_docker_analyzer_get))
+    @mock_connections(patch("requests.post", side_effect=mocked_docker_analyzer_post))
+    def test_boxjs(self, mock_get=None, mock_post=None):
+        report = boxjs_scan.BoxJS(
+            "BoxJS_Scan_JavaScript",
+            self.job_id,
+            self.filepath,
+            self.filename,
+            self.md5,
+            {},
+        ).start()
+        self.assertEqual(report.get("success", False), True)
+
+
 def _generate_test_job_with_file(params, filename):
     test_file = f"{settings.PROJECT_LOCATION}/test_files/{filename}"
     with open(test_file, "rb") as f: