Better logging, CI , Template bug fix and PEP-8 (#856)

* improve logging
* Travis CI
* Fix breaking Templates

.gitignore

@@ -71,3 +71,6 @@ secret
 
 #IntelliJ IDE
 .idea
+*/migrations/*
+.fuse_hidden*
+/MalwareAnalyzer/malwaredb/*

.travis.yml

@@ -0,0 +1,56 @@
+dist: xenial
+language: python
+python:
+- '3.6'
+virtualenv:
+  system_site_packages: false
+sudo: false
+cache:
+- apt
+- pip
+before_install:
+- export TRAVIS_COMMIT_MSG="$(git log --format=%B --no-merges -n 1)"
+- REPO=`pwd`
+- echo $REPO
+- echo $TRAVIS_TAG
+- pwd
+- export PDFGEN_PKGFILE="wkhtmltox_0.12.5-1.xenial_amd64.deb"
+- export PDFGEN_URL="https://downloads.wkhtmltopdf.org/0.12/0.12.5/${PDFGEN_PKGFILE}"
+- export YARA_URL="https://github.com/rednaga/yara-python-1"
+- export POSTGRES=False
+- export LIBSSL1="http://security.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.0g-2ubuntu4.3_amd64.deb"
+- export LIBSSL1_NAME="libssl1.1_1.1.0g-2ubuntu4.3_amd64.deb"
+- sudo apt-get install -y libpng16-16 libssl-dev xfonts-75dpi xfonts-base libffi-dev libxml2-dev libxslt1-dev
+- sudo apt-get install -y openssl
+- wget ${PDFGEN_URL}
+- wget ${LIBSSL1}
+- sudo dpkg -i ${LIBSSL1_NAME}
+- sudo dpkg -i ${PDFGEN_PKGFILE}
+- sudo apt install -y software-properties-common
+- sudo add-apt-repository ppa:linuxuprising/java -y
+- sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 762E3157
+- sudo apt update
+- echo oracle-java11-installer shared/accepted-oracle-license-v1-2 select true | sudo /usr/bin/debconf-set-selections
+- sudo apt install -y oracle-java11-installer
+addons:
+  apt:
+    packages:
+      - python3-pip
+      - git
+      - unzip
+      - python3-dev
+      - libevent-dev
+      - libxml2
+      - libxml2-dev
+      - libxml2-utils
+      - software-properties-common
+      - python3-setuptools
+      - sqlite3
+install:
+- export PYTHONIOENCODING=UTF8
+- pip install coveralls
+before_script:
+  # apkid is not being tested currently
+  - pip install -r requirements.txt
+script:
+  - python manage.py test

DynamicAnalyzer/tools/webproxy.py

@@ -17,7 +17,7 @@ def stop_capfuzz(port):
     # Invoke CapFuzz UI Kill Request
     try:
         requests.get("http://127.0.0.1:" + str(port) + "/kill", timeout=5)
-        print("[INFO] Killing CapFuzz UI")
+        logger.info("Killing CapFuzz UI")
     except:
         pass
 
@@ -28,7 +28,7 @@ def stop_capfuzz(port):
         url = "http://127.0.0.1"
         requests.get(url, headers=headers, proxies={
                      'http': http_proxy})
-        print("[INFO] Killing CapFuzz Proxy")
+        logger.info("Killing CapFuzz Proxy")
     except:
         pass
 

DynamicAnalyzer/views/android/analysis.py

@@ -23,7 +23,7 @@
 def api_analysis(package, location):
     """API Analysis"""
     api_analysis_result = {}
-    print("\n[INFO] Dynamic API Analysis")
+    logger.info("Dynamic API Analysis")
     dat = ""
     api_base64 = []
     api_fileio = []
@@ -128,7 +128,7 @@ def api_analysis(package, location):
 def run_analysis(apk_dir, md5_hash, package):
     """Run Dynamic File Analysis"""
     analysis_result = {}
-    print("\n[INFO] Dynamic File Analysis")
+    logger.info("Dynamic File Analysis")
     capfuzz_home = os.path.join(str(Path.home()), ".capfuzz")
     web = os.path.join(capfuzz_home, 'flows', package + ".flows.txt")
     logcat = os.path.join(apk_dir, 'logcat.txt')
@@ -159,7 +159,7 @@ def run_analysis(apk_dir, md5_hash, package):
     url_pattern = re.compile(r'((?:https?://|s?ftps?://|file://|javascript:|data:|www\d{0,3}[.])[\w().=/;,#:@?&~*+!$%\'{}-]+)', re.UNICODE)
     urllist = re.findall(url_pattern, traffic.lower())
     # Domain Extraction and Malware Check
-    print("[INFO] Performing Malware Check on extracted Domains")
+    logger.info("Performing Malware Check on extracted Domains")
     domains = malware_check(urllist)
     for url in urllist:
         if url not in urls:
@@ -240,7 +240,7 @@ def run_analysis(apk_dir, md5_hash, package):
 
 def download(md5_hash, download_dir, apk_dir, package):
     """Generating Downloads"""
-    print("\n[INFO] Generating Downloads")
+    logger.info("Generating Downloads")
     try:
 
         capfuzz_home = os.path.join(str(Path.home()), ".capfuzz")

DynamicAnalyzer/views/android/avd.py

@@ -5,16 +5,17 @@
 import time
 import platform
 import subprocess
-
+import logging
 from DynamicAnalyzer.views.android.shared import adb_command
 from MobSF.utils import PrintException
 from django.conf import settings
 from scripts.start_avd import main as start_avd_cold
+logger = logging.getLogger(__name__)
 
 
 def stop_avd():
     """Stop AVD"""
-    print("\n[INFO] Stopping MobSF Emulator")
+    logger.info("Stopping MobSF Emulator")
     try:
         adb_command(['emu', 'kill'], silent=True)
     except:
@@ -23,7 +24,7 @@ def stop_avd():
 
 def start_avd_from_snapshot():
     """Start AVD"""
-    print("\n[INFO] Starting MobSF Emulator")
+    logger.info("Starting MobSF Emulator")
     try:
         if platform.system() == 'Darwin':
             # There is a strage error in mac with the dyld one in a while..
@@ -70,25 +71,25 @@ def refresh_avd():
     for path in [settings.AVD_EMULATOR,
                  settings.ADB_BINARY]:
         if not path:
-            print("\n[ERROR] AVD binaries not configured, please refer to the official documentation")
+            logger.error("AVD binaries not configured, please refer to the official documentation")
             return False
 
-    print("\n[INFO] Refreshing MobSF Emulator")
+    logger.info("Refreshing MobSF Emulator")
     try:
         # Stop existing emulator
         stop_avd()
 
         # Check if configuration specifies cold or warm boot
         if settings.AVD_COLD_BOOT:
             if start_avd_cold():
-                print("\n[INFO] AVD has been started successfully")
+                logger.info("AVD has been started successfully")
                 return True
         else:
             if not settings.AVD_SNAPSHOT:
-                print("\n[ERROR] AVD not configured properly - AVD_SNAPSHOT is missing")
+                logger.error("AVD not configured properly - AVD_SNAPSHOT is missing")
                 return False
             if start_avd_from_snapshot():
-                print("\n[INFO] AVD has been loaded from snapshot successfully")
+                logger.info("AVD has been loaded from snapshot successfully")
                 return True
         return False