Merge pull request #26 from xabiugarte/dev

Dev

Dockerfile

@@ -1,9 +1,9 @@
 FROM ubuntu
 RUN apt-get update
 RUN apt-get -y dist-upgrade
-RUN apt-get install -y python-pip mysql-client libmysqlclient-dev apache2 libapache2-mod-wsgi libapache2-mod-wsgi
+RUN apt-get install -y python3-pip mysql-client libmysqlclient-dev apache2 libapache2-mod-wsgi-py3
 COPY install/requirements.txt /tmp
-RUN pip install -r /tmp/requirements.txt && rm /tmp/requirements.txt
+RUN pip3 install -r /tmp/requirements.txt && rm /tmp/requirements.txt
 
 RUN useradd -m -U -d /home/first -s /bin/bash first
 COPY ./server /home/first

docs/index.rst

@@ -68,6 +68,10 @@ Once an engine is installed you can start using your FIRST installation to add a
         $ cd FIRST-server/server
         $ python manage.py runserver 0.0.0.0:1337
 
+.. note:: FreeBSD port
+
+    FIRST also has a FreeBSD port available: https://www.freshports.org/security/py-first-server/
+
 .. _server-docs:
 
 .. toctree::

install/run.sh

@@ -21,7 +21,7 @@ else
 fi
 
 # Always run migrations
-/usr/bin/python /home/first/manage.py migrate
+/usr/bin/python3 /home/first/manage.py migrate
 
 # Finally, start up the apache service
 /usr/sbin/apache2ctl -D FOREGROUND

server/example_config.json

@@ -11,5 +11,5 @@
     "debug" : true,
     "allowed_hosts" : ["localhost", "testserver"],
 
-    "oauth_path" : "",
+    "oauth_path" : "/path/to/your/google_secret.json",
 }

server/first/settings.py

@@ -19,13 +19,14 @@
                                 'first_config.json')
 CONFIG = {}
 try:
-    config_data = json.load(file(FIRST_CONFIG_FILE))
+    with open(FIRST_CONFIG_FILE, "r") as f:
+        config_data = json.load(f)
     if type(config_data) == dict:
         CONFIG = config_data
 except IOError as ioe:
-    print '[1st] IOError: {}'.format(ioe)
+    print('[1st] IOError: {}'.format(ioe))
 except ValueError as ve:
-    print '[1st] ValueError: {}'.format(ve)
+    print('[1st] ValueError: {}'.format(ve))
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

server/first/urls.py

@@ -15,12 +15,12 @@
 """
 from django.contrib import admin
 from django.conf.urls import handler404
-from django.conf.urls import include, url
+from django.urls import path, re_path, include
 
 handler404 = 'www.views.handler404'
 
 urlpatterns = [
-    url(r'^admin/', admin.site.urls),
-    url(r'^api/', include('rest.urls')),
-    url(r'^', include('www.urls')),
+    path(r'admin', admin.site.urls),
+    path(r'api', include('rest.urls')),
+    path(r'', include('www.urls')),
 ]

server/first_core/auth.py

@@ -127,7 +127,7 @@ def __init__(self, request):
                                     redirect_uri=redirect_uri),
                 }
         except TypeError as e:
-            print e
+            print(e)
 
         if 'auth' not in request.session:
             request.session['auth'] = {}
@@ -190,9 +190,9 @@ def login_step_2(self, auth_code, url, login=True):
 
                 if login:
                     try:
-                        user = User.objects.get(email=email)
+                        user = User.objects.get(email=self.request.session['info']['email'])
                         user.auth_data = json.dumps(credentials)
-                        user.name = info['displayName']
+                        user.name = self.request.session['info']['name']
                         user.save()
 
                         self.request.session['auth']['api_key'] = str(user.api_key)
@@ -241,7 +241,7 @@ def register_user(self):
             #   Create random 4 digit value for the handle
             #   This prevents handle collisions
             number = random.randint(0, 9999)
-            for i in xrange(10000):
+            for i in range(10000):
                 try:
                     num = (number + i) % 10000
                     user = User.objects.get(handle=handle, number=num)

server/first_core/dbs/__init__.py

@@ -20,7 +20,7 @@
 #-------------------------------------------------------------------------------
 
 #   Python Modules
-import ConfigParser
+import configparser 
 from hashlib import md5
 
 #   FIRST Modules
@@ -58,7 +58,7 @@ def __init__(self, conf):
         '''
         Constructor.
 
-        @param conf: ConfigParser.RawConfigParser
+        @param conf: configparser.RawConfigParser
         '''
         raise FIRSTDBError('TODO: implement')
 
@@ -79,10 +79,10 @@ def __init__(self, config=None):
                     self._dbs[d.name] = d
 
             except FIRSTDBError as e:
-                print e
+                print(e)
 
         if not self._dbs:
-            print '[DBM] Error: No dbs could be loaded'
+            print('[DBM] Error: No dbs could be loaded')
 
     def db_list(self):
         '''

server/first_core/dbs/builtin_db.py

@@ -28,7 +28,7 @@
 import math
 import json
 import hashlib
-import ConfigParser
+import configparser 
 from hashlib import md5
 
 #   Third Party Modules
@@ -58,7 +58,7 @@ def __init__(self, config):
         '''
         Constructor.
 
-        @param conf: ConfigParser.RawConfigParser
+        @param conf: configparser.RawConfigParser
         '''
         self._is_installed = True
         '''
@@ -169,7 +169,7 @@ def get_function(self, opcodes, architecture, apis, create=False, **kwargs):
                                                     opcodes=opcodes,
                                                     architecture=architecture)
 
-                apis_ = [FunctionApis.objects.get_or_create(x)[0] for x in apis]
+                apis_ = [FunctionApis.objects.get_or_create(api=x)[0] for x in apis]
                 for api in apis_:
                     function.apis.add(api)
 
@@ -241,7 +241,7 @@ def get_metadata_list(self, metadata):
         results = []
         metadata_ids, engine_metadata = separate_metadata(metadata)
 
-        for _id, metadata in Metadata.objects.in_bulk(metadata_ids).iteritems():
+        for _id, metadata in Metadata.objects.in_bulk(metadata_ids).items():
             data = metadata.dump()
             data['id'] = make_id(0, metadata=metadata.id)
             results.append(data)
@@ -303,7 +303,7 @@ def metadata_history(self, metadata):
         e_comment = ('Generated by Engine: {0.name}\n{0.description}\n\n'
                     'Developer: {0.developer.user_handle}')
 
-        for _id, metadata in Metadata.objects.in_bulk(metadata_ids).iteritems():
+        for _id, metadata in Metadata.objects.in_bulk(metadata_ids).items():
             data = metadata.dump(True)
             result_key = make_id(0, metadata=_id)
             results[result_key] = { 'creator' : data['creator'],

server/first_core/disassembly/__init__.py

@@ -123,7 +123,7 @@ def __init__(self, architecture, code):
 
     def instructions(self):
         #   When first called function will return cached instructions
-        for i in xrange(len(self.data)):
+        for i in range(len(self.data)):
             yield self.data[i]
 
         #   Then iterate through non-cached instructions

server/first_core/engines/__init__.py

@@ -13,6 +13,7 @@
 #   Python Modules
 import re
 import sys
+import functools
 
 #   First Modules
 from first_core.error import FIRSTError
@@ -90,8 +91,8 @@ def __init__(self, dbs, engine_id, rank):
     def add(self, function):
         required_keys = {'id', 'apis', 'opcodes', 'architecture', 'sha256'}
         if ((dict != type(function))
-            or not required_keys.issubset(function.keys())):
-            print 'Data provided is not the correct type or required keys not provided'
+            or not required_keys.issubset(list(function.keys()))):
+            print('Data provided is not the correct type or required keys not provided')
             return
 
         self._add(function)
@@ -113,7 +114,7 @@ def install(self):
         try:
             self._install()
         except FIRSTEngineError as e:
-            if e.message == 'Not Implemented':
+            if str(e) == 'Not Implemented':
                 return
 
             raise e
@@ -122,7 +123,7 @@ def uninstall(self):
         try:
             self._uninstall()
         except FIRSTEngineError as e:
-            if e.message == 'Not Implemented':
+            if str(e) == 'Not Implemented':
                 return
 
             raise e
@@ -185,17 +186,17 @@ def _engines(self):
             try:
                 e = obj(self.__db_manager, str(e.id), e.rank)
                 if not isinstance(e, AbstractEngine):
-                    print '[EM] {} is not an AbstractEngine'.format(e)
+                    print('[EM] {} is not an AbstractEngine'.format(e))
                     continue
 
                 if e.is_operational:
                     engines.append(e)
 
             except FIRSTEngineError as e:
-                print e
+                print(e)
 
         if not engines:
-            print '[EM] Error: No engines could be loaded'
+            print('[EM] Error: No engines could be loaded')
 
         return engines
 
@@ -217,8 +218,8 @@ def add(self, function):
 
         '''
         required_keys = {'id', 'apis', 'opcodes', 'architecture', 'sha256'}
-        if (dict != type(function)) or not required_keys.issubset(function.keys()):
-            print '[1stEM] Data provided is not the correct type or required keys not provided'
+        if (dict != type(function)) or not required_keys.issubset(list(function.keys())):
+            print('[1stEM] Data provided is not the correct type or required keys not provided')
             return None
 
         dis = Disassembly(function['architecture'], function['opcodes'])
@@ -270,7 +271,7 @@ def scan(self, user, opcodes, architecture, apis):
         engines = self._engines
 
         dis = Disassembly(architecture, opcodes)
-        for i in xrange(len(engines)):
+        for i in range(len(engines)):
             engine = engines[i]
             try:
                 results = engine.scan(opcodes, architecture, apis,
@@ -279,10 +280,10 @@ def scan(self, user, opcodes, architecture, apis):
                     engine_results[i] = results
 
             except Exception as e:
-                print e
+                print(e)
 
         results = {}
-        for i, hits in engine_results.iteritems():
+        for i, hits in engine_results.items():
             engine = engines[i]
 
             for result in hits:
@@ -297,8 +298,8 @@ def scan(self, user, opcodes, architecture, apis):
                     results[result.id].similarity = result.similarity
 
         #   Order functions
-        cmp_func = lambda x,y: cmp(y.similarity, x.similarity)
-        ordered_functions = sorted(results.values(), cmp_func)
+        cmp_func = lambda x,y: (y.similarity > x.similarity) - (y.similarity < x.similarity)
+        ordered_functions = sorted(results.values(), key=functools.cmp_to_key(cmp_func))
 
         #   Create Metadata list
         #   TODO: Narrow results to top 20 hits, use similarity and metadata rank

server/first_core/engines/basic_masking.py

@@ -79,7 +79,8 @@ def normalize(self, disassembly):
             normalized = []
             original = []
             for i in disassembly.instructions():
-                original.append(str(i.bytes).encode('hex'))
+                import codecs
+                original.append(codecs.encode(i.bytes, 'hex'))
                 instr = ''.join(chr(x) for x in i.opcode if x)
 
                 #   Special mnemonic masking (Call, Jmp, JCC)
@@ -147,14 +148,14 @@ def normalize(self, disassembly):
             '''
 
             if MIN_REQUIRED_INSTRUCTIONS > len(normalized):
-                print 145
+                print(145)
                 return (0, None)
 
-            h_sha256 = sha256(''.join(normalized)).hexdigest()
+            h_sha256 = sha256(''.join(normalized).encode('utf-8')).hexdigest()
             return (changed_bytes, h_sha256)
 
         except Exception as e:
-            print 160, e
+            print(160, e)
 
             return (0, None)
 
@@ -248,4 +249,4 @@ def _install(self):
         execute_from_command_line(['manage.py', 'migrate', 'engines'])
 
     def _uninstall(self):
-        print 'Manually delete tables associated with {}'.format(self.engine_name)
+        print('Manually delete tables associated with {}'.format(self.engine_name))

server/first_core/engines/exact_match.py

@@ -50,7 +50,7 @@ def _scan(self, opcodes, architecture, apis, disassembly):
             return None
 
         similarity = 90.0
-        if set(function.apis.values()) == set(apis):
+        if set([el['api'] for el in function.apis.values()]) == set(apis):
             similarity += 10.0
 
         return [FunctionResult(str(function.id), similarity)]

server/first_core/engines/mnemonic_hash.py

@@ -80,7 +80,7 @@ def mnemonic_hash(self, disassembly):
             if len(mnemonics) < MIN_REQUIRED_MNEMONICS:
                 return (None, None)
 
-            return (mnemonics, sha256(''.join(mnemonics)).hexdigest())
+            return (mnemonics, sha256(''.join(mnemonics).encode('utf-8')).hexdigest())
 
         except Exception as e:
             raise e
@@ -165,4 +165,4 @@ def _install(self):
         execute_from_command_line(['manage.py', 'migrate', 'engines'])
 
     def _uninstall(self):
-        print 'Manually delete tables associated with {}'.format(self.engine_name)
+        print('Manually delete tables associated with {}'.format(self.engine_name))

server/first_core/engines/results.py

@@ -27,8 +27,8 @@ def get_metadata(self, db):
         while True:
             data = self._get_metadata(db)
 
-            if ((type(data) != dict) or (None in data.values())
-                or (not Result.required.issubset(data.keys()))):
+            if ((type(data) != dict) or (None in list(data.values()))
+                or (not Result.required.issubset(list(data.keys())))):
                 return
 
             data['similarity'] = self.similarity

server/first_core/util.py

@@ -33,7 +33,7 @@ def make_id(flags, metadata=0, engine=0):
         string: A 26 byte hex string
     '''
     data = [flags, metadata, engine]
-    if (None in data) or (not all([type(x) in [int, long] for x in data])):
+    if (None in data) or (not all([type(x) in [int] for x in data])):
         return None
 
     if ((engine > (2**32 - 1)) or (metadata > (2**64 - 1))
@@ -44,13 +44,13 @@ def make_id(flags, metadata=0, engine=0):
 
 
 def parse_id(_id):
-    if type(_id) in [str, unicode]:
+    if type(_id) in [str]:
         if len(_id) != 26:
             return (None, None, None)
 
         _id = int(_id, 16)
 
-    elif type(id) not in [int, long]:
+    elif type(id) not in [int]:
         return (None, None, None)
 
     flag = _id >> (8 * 12)