Merge pull request #156 from intelowlproject/develop

Quark, Pulsedive, Python 3.7, Docs update, Fix in active_dns

.dockerignore

@@ -1,10 +1,20 @@
 .gitignore
+.git
 .vscode
+.lgtm.yml
+.travis.yml
 __pycache__
+venv/
 .env
 env_file_app
+env_file_app_template
 env_file_postgres
+env_file_postgres_template
 env_file_integrations
-venv/
+env_file_integrations_template
+env_file_app_travis
+docs/
+integrations/
 settings/ldap_config.py
-docker-compose-override.yml
+docker-compose*
+*.quark.log

.env

@@ -2,10 +2,10 @@
 ### All services specified in all compose-files in variable COMPOSE_FILE will be built/ran
 ### By default, when you use `docker-compose up` only docker-compose.yml is read
 ### For each additional integration, the location of it's docker-compose.<>.yml file should be appended to 
-### the COMPOSE_FILE variable each seperated with ':'. If you are on windows, replace all ':' with ';'.
+### the COMPOSE_FILE variable each separated with ':'. If you are on windows, replace all ':' with ';'.
 ### Reference to Docker's official Docs: https://docs.docker.com/compose/reference/envvars/#compose_file#compose_file
 
-INTELOWL_TAG_VERSION=v1.3.1
+INTELOWL_TAG_VERSION=v1.4.0
 
 ###### Default (Production) ######
 
@@ -25,3 +25,4 @@ COMPOSE_FILE=docker-compose.yml
 ###### For travis ######
 
 #COMPOSE_FILE=docker-compose-for-tests.yml:./integrations/docker-compose-for-tests.peframe.yml:./integrations/docker-compose-for-tests.thug.yml:./integrations/docker-compose-for-tests.capa.yml:./integrations/docker-compose-for-tests.boxjs.yml:./integrations/docker-compose-for-tests.apk.yml
+

.gitignore

@@ -7,4 +7,4 @@ env_file_postgres
 env_file_integrations
 venv/
 settings/ldap_config.py
-docker-compose-override.yml
+docker-compose-override.yml

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.6
+FROM python:3.7
 
 ENV PYTHONUNBUFFERED 1
 ENV DJANGO_SETTINGS_MODULE intel_owl.settings
@@ -35,6 +35,7 @@ RUN touch ${LOG_PATH}/django/api_app.log ${LOG_PATH}/django/api_app_errors.log \
 # this is cause stringstifer creates this directory during the build and cause celery to crash
     && rm -rf /root/.local
 
+# download yara rules and quark engine rules
 RUN api_app/script_analyzers/yara_repo_downloader.sh
 
 # this is because botocore points to legacy endpoints

README.md

@@ -40,6 +40,8 @@ Documentation about IntelOwl installation, usage, contribution can be found at h
 
 [First announcement](https://www.certego.net/en/news/new-year-new-tool-intel-owl/)
 
+[Daily Swig Article](https://portswigger.net/daily-swig/intel-owl-osint-tool-automates-the-intel-gathering-process-using-a-single-api)
+
 ### Free Internal Modules Available
 
 - Static Doc Analysis
@@ -143,14 +145,14 @@ This project was created and will be upgraded thanks to the following organizati
 
 ### Google Summer Of Code
 
-The project was accepted to the GSoC 2020 under the Honeynet Project!!
-
-Stay tuned for upcoming [new features](https://www.honeynet.org/gsoc/gsoc-2020/google-summer-of-code-2020-project-ideas/#intel-owl-improvements) developed by Eshaan Bansal ([Twitter](https://twitter.com/mask0fmydisguis)).
+The project was accepted to the GSoC 2020 under the Honeynet Project!! A lot of [new features](https://www.honeynet.org/gsoc/gsoc-2020/google-summer-of-code-2020-project-ideas/#intel-owl-improvements) were developed by Eshaan Bansal ([Twitter](https://twitter.com/mask0fmydisguis)).
 
-### About the author
+Stay tuned for the upcoming GSoC 2021! Join the [Honeynet Slack chat](https://gsoc-slack.honeynet.org/) for more info.
 
-Feel free to contact the author at any time:
-Matteo Lodi ([Twitter](https://twitter.com/matte_lodi))
+### About the author and maintainers
 
+Feel free to contact the main developers at any time:
+- Matteo Lodi ([Twitter](https://twitter.com/matte_lodi)): Author and creator
+- Eshaan Bansal ([Twitter](https://twitter.com/mask0fmydisguis)): Principal maintainer
 
 We also have a dedicated twitter account for the project: [@intel_owl](https://twitter.com/intel_owl).

api_app/models.py

@@ -28,13 +28,18 @@ def __str__(self):
 
 
 class Job(models.Model):
+    class Meta:
+        indexes = [
+            models.Index(fields=["md5", "status",]),
+        ]
+
     source = models.CharField(max_length=50, blank=False, default="none")
     is_sample = models.BooleanField(blank=False, default=False)
     md5 = models.CharField(max_length=32, blank=False)
     observable_name = models.CharField(max_length=512, blank=True)
     observable_classification = models.CharField(max_length=12, blank=True)
     file_name = models.CharField(max_length=50, blank=True)
-    file_mimetype = models.CharField(max_length=50, blank=True)
+    file_mimetype = models.CharField(max_length=80, blank=True)
     status = models.CharField(
         max_length=32, blank=False, choices=STATUS, default="pending"
     )

api_app/script_analyzers/classes.py

@@ -3,7 +3,7 @@
 import logging
 import requests
 import json
-from abc import ABC, abstractmethod
+from abc import ABCMeta, abstractmethod
 
 from api_app.exceptions import (
     AnalyzerRunNotImplemented,
@@ -15,7 +15,7 @@
 logger = logging.getLogger(__name__)
 
 
-class BaseAnalyzerMixin(ABC):
+class BaseAnalyzerMixin(metaclass=ABCMeta):
     """
     Abstract Base class for Analyzers.
     Never inherit from this branch,
@@ -177,7 +177,7 @@ def after_run(self):
         )
 
 
-class DockerBasedAnalyzer(ABC):
+class DockerBasedAnalyzer(metaclass=ABCMeta):
     """
     Abstract class for a docker based analyzer (integration).
     Inherit this branch along with either one of ObservableAnalyzer or FileAnalyzer
@@ -279,7 +279,7 @@ def _docker_run(self, req_data, req_files=None):
         """
 
         # handle in case this is a test
-        if hasattr(self, "is_test"):
+        if hasattr(self, "is_test") and getattr(self, "is_test"):
             # only happens in case of testing
             self.report["success"] = True
             return {}

api_app/script_analyzers/file_analyzers/quark_engine.py

@@ -0,0 +1,13 @@
+from api_app.script_analyzers.classes import FileAnalyzer
+
+
+class QuarkEngine(FileAnalyzer):
+    def run(self):
+        from quark.report import Report
+
+        # new report object
+        report = Report()
+        # start analysis
+        report.analysis(self.filepath, "/opt/deploy/quark-rules")
+        # return json report
+        return report.get_report("json")

api_app/script_analyzers/observable_analyzers/active_dns.py

@@ -68,6 +68,7 @@ def __handle_activedns_error(self, err: str):
         self.report["success"] = False
 
     def __doh_google(self):
+        result = {}
         if self.observable_classification == "domain":
             try:
                 authority_answer = ""
@@ -94,7 +95,7 @@ def __doh_google(self):
                             f"observable: {self.observable_name} active_dns query"
                             f" retrieved no valid A answer: {answers}"
                         )
-                self.report["report"] = {
+                result = {
                     "name": self.observable_name,
                     "resolution": ip,
                     "authoritative_answer": authority_answer,
@@ -107,8 +108,10 @@ def __doh_google(self):
             self.__handle_activedns_error(
                 "cannot analyze something different from type: domain"
             )
+        return result
 
     def __doh_cloudflare(self):
+        result = {}
         if self.observable_classification == "domain":
             try:
                 client = requests.session()
@@ -131,7 +134,7 @@ def __doh_cloudflare(self):
                     else "NXDOMAIN"
                 )
 
-                self.report["report"] = {
+                result = {
                     "name": self.observable_name,
                     "resolution": result_data,
                 }
@@ -143,8 +146,10 @@ def __doh_cloudflare(self):
             self.__handle_activedns_error(
                 "cannot analyze something different from type: domain"
             )
+        return result
 
     def __doh_cloudflare_malware(self):
+        result = {}
         if self.observable_classification == "domain":
             try:
                 result = {"name": self.observable_name}
@@ -169,14 +174,15 @@ def __doh_cloudflare_malware(self):
                     # known as malicious
                     if resolution == "0.0.0.0":
                         result["is_malicious"] = True
+                    else:
+                        result["is_malicious"] = False
                 else:
                     logger.warning(
                         f"no Answer key retrieved for {self.observable_name}"
                         f"DNS request coming from {self.analyzer_name} analyzer"
                     )
                     result["no_answer"] = True
 
-                self.report["report"] = result
             except requests.exceptions.RequestException as err:
                 self.__handle_activedns_error(
                     f"observable_name:{self.observable_name}, RequestException {err}"
@@ -185,6 +191,7 @@ def __doh_cloudflare_malware(self):
             self.__handle_activedns_error(
                 "cannot analyze something different from type: domain"
             )
+        return result
 
     def __classic_dns(self):
         result = {}
@@ -193,11 +200,14 @@ def __classic_dns(self):
             resolutions = []
             try:
                 domains = socket.gethostbyaddr(self.observable_name)
-                resolutions = domains[2]
+                resolutions = domains[0]
             except (socket.gaierror, socket.herror):
                 logger.info(
                     f"no resolution found for observable {self.observable_name}"
                 )
+            logger.info(
+                f"resolution {resolutions} found for observable {self.observable_name}"
+            )
             result = {"name": self.observable_name, "resolutions": resolutions}
         elif self.observable_classification == "domain":
             try:
@@ -208,4 +218,4 @@ def __classic_dns(self):
         else:
             self.__handle_activedns_error("not analyzable")
 
-        self.report["report"] = result
+        return result

api_app/script_analyzers/observable_analyzers/pulsedive.py

@@ -0,0 +1,92 @@
+import logging
+import requests
+import time
+
+from api_app.exceptions import AnalyzerRunException
+from api_app.script_analyzers.classes import ObservableAnalyzer
+from intel_owl import secrets
+
+
+logger = logging.getLogger(__name__)
+
+
+class Pulsedive(ObservableAnalyzer):
+    base_url: str = "https://pulsedive.com/api"
+    max_tries: int = 10
+    poll_distance: int = 10
+
+    def set_config(self, additional_config_params):
+        self.api_key_name = additional_config_params.get(
+            "api_key_name", "PULSEDIVE_API_KEY"
+        )
+        self.__api_key = secrets.get_secret(self.api_key_name)
+        active_scan = additional_config_params.get("active_scan", True)
+        self.probe = 1 if active_scan else 0
+
+    def run(self):
+        result = {}
+        if not self.__api_key:
+            warning = f"No API key retrieved with name: {self.api_key_name}"
+            logger.info(
+                f"{warning}. Continuing without API key..." f" <- {self.__repr__()}"
+            )
+            self.report["errors"].append(warning)
+        else:
+            default_param = f"&key={self.__api_key}"
+
+        # headers = {"Key": self.__api_key, "Accept": "application/json"}
+        # 1. query to info.php to check if the indicator is already in the database
+        params = f"indicator={self.observable_name}"
+        if self.__api_key:
+            params += default_param
+        resp = requests.get(f"{self.base_url}/info.php?{params}")
+        resp.raise_for_status()
+        result = resp.json()
+        e = result.get("error", None)
+        if e == "Indicator not found.":
+            # 2. submit new scan to analyze.php
+            params = f"value={self.observable_name}&probe={self.probe}"
+            if self.__api_key:
+                params += default_param
+            headers = {
+                "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8"
+            }
+            resp = requests.post(
+                f"{self.base_url}/analyze.php", data=params, headers=headers
+            )
+            resp.raise_for_status()
+            qid = resp.json().get("qid", None)
+            # 3. retrieve result using qid after waiting for 10 seconds
+            params = f"qid={qid}"
+            if self.__api_key:
+                params += default_param
+            result = self.__poll_for_result(params)
+            if result.get("data", None):
+                result = result["data"]
+
+        return result
+
+    def __poll_for_result(self, params):
+        result = {}
+        url = f"{self.base_url}/analyze.php?{params}"
+        obj_repr = self.__repr__()
+        for chance in range(self.max_tries):
+            logger.info(
+                f"polling request #{chance+1} for observable: {self.observable_name}"
+                f" <- {obj_repr}"
+            )
+            time.sleep(self.poll_distance)
+            resp = requests.get(url)
+            resp.raise_for_status()
+            resp_json = resp.json()
+            status = resp_json.get("status", None)
+            if status == "done":
+                result = resp_json
+                break
+            elif status == "processing":
+                continue
+            else:
+                err = resp_json.get("error", "Report not found.")
+                raise AnalyzerRunException(err)
+
+        return result

api_app/script_analyzers/observable_analyzers/shodan.py

@@ -1,6 +1,6 @@
 import requests
 
-from api_app.exceptions import AnalyzerRunException
+from api_app.exceptions import AnalyzerRunException, AnalyzerConfigurationException
 from api_app.script_analyzers import classes
 from intel_owl import secrets
 
@@ -10,12 +10,14 @@ class Shodan(classes.ObservableAnalyzer):
 
     def set_config(self, additional_config_params):
         self.analysis_type = additional_config_params.get("shodan_analysis", "search")
-        api_key_name = additional_config_params.get("api_key_name", "SHODAN_KEY")
-        self.__api_key = secrets.get_secret(api_key_name)
+        self.api_key_name = additional_config_params.get("api_key_name", "SHODAN_KEY")
+        self.__api_key = secrets.get_secret(self.api_key_name)
 
     def run(self):
         if not self.__api_key:
-            raise AnalyzerRunException("no api key retrieved")
+            raise AnalyzerConfigurationException(
+                f"No API key retrieved with name: {self.api_key_name}."
+            )
 
         if self.analysis_type == "search":
             params = {"key": self.__api_key, "minify": True}
@@ -26,9 +28,9 @@ def run(self):
             }
             uri = f"labs/honeyscore/{self.observable_name}"
         else:
-            raise AnalyzerRunException(
-                f"not supported observable type {self.observable_classification}."
-                "Supported is IP"
+            raise AnalyzerConfigurationException(
+                f"analysis type: '{self.analysis_type}' not suported."
+                "Supported are: 'search', 'honeyscore'."
             )
 
         try: