Travis CI will now check for coding conventions

.travis.yml

@@ -0,0 +1,11 @@
+language: python
+python:
+  - "2.7"
+dist: xenial
+sudo: true
+cache: pip
+install:
+  - pip install -r requirements.txt
+script:
+  - flake8 elementals
+  - python tests.py

requirements.txt

@@ -0,0 +1,4 @@
+elementals
+sark
+pydocstyle
+flake8

setup.cfg

@@ -0,0 +1,3 @@
+[flake8]
+exclude = .git,__init__.py
+ignore = E272, E501, E502, E221, E302, E241, E126, E127, E128, E266, E731, F405, F403

setup.py

@@ -9,16 +9,17 @@
 
 setup(name='Karta',
       version='0.9.0',
-      description='IDA plugin for matching open-source libraries in huge binaries',
+      description='IDA plugin for identifying and matching open-source libraries in (huge) binaries',
       author='Eyal Itkin',
       author_email='eyalit@checkpoint.com',
       long_description=long_description,
       long_description_content_type="text/markdown",
+	  url='https://github.com/CheckPointSW/Karta',
       license='MIT',
       packages=find_packages(),
-      install_requires=['elementals', 'sark'],
+      install_requires=['elementals', 'sark', 'pydocstyle', 'flake8'],
       classifiers=[
-                    "Programming Language :: Python",
+                    "Programming Language :: Python :: 2",
                     "License :: OSI Approved :: MIT License (MIT License)",
                     "Operating System :: OS Independent",
                   ],

src/analyze_src_file.py

@@ -5,8 +5,8 @@
 import logging
 import traceback
 
-def analyzeFile() :
-    """Analyzes all of the (source) functions for a single compiled file"""
+def analyzeFile():
+    """Analyzes all of the (source) functions for a single compiled file."""
     disas = getDisas()
     logger.info("Started the Script")
     contexts = []
@@ -16,20 +16,21 @@ def analyzeFile() :
         setWindowsMode()
     # build the list of exported (non-static) functions
     exported = disas.exports()
-    for segment_idx in xrange(disas.numSegments()) :
+    for segment_idx in xrange(disas.numSegments()):
         if ".text" not in disas.segmentName(segment_idx):
             continue
-        for function_ea in disas.segmentFunctions(segment_idx) :
+        for function_ea in disas.segmentFunctions(segment_idx):
             src_ctx = disas.analyzeFunction(function_ea, True)
             # check if static or not
-            if src_ctx.name not in exported :
+            if src_ctx.name not in exported:
                 src_ctx.markStatic()
             contexts.append(src_ctx)
     functionsToFile(disas.inputFile(), contexts)
     logger.info("Finished Successfully")
 
+
 # create a logger
-logger = Logger(LIBRARY_NAME, use_stdout = False)
+logger = Logger(LIBRARY_NAME, use_stdout=False)
 # Always init the utils before we start
 initUtils(logger, createDisassemblerHandler(logger))
 # Register our contexts

src/ar_parser.py

@@ -1,13 +1,13 @@
 import os
 
-def getArchiveFiles(ar_path) :
-    """Returns an orderred list of the files within the .ar archive
-    
+def getArchiveFiles(ar_path):
+    """Return an ordered list of the files within the .ar archive.
+
     Args:
         ar_path (str): path to the compiled *.ar file
 
     Return Value:
-        Orderred list of file names
+        Ordered list of file names
     """
     ar_fd = open(ar_path, 'rb')
     is_windows = ar_path.endswith(".lib")
@@ -26,15 +26,15 @@ def getArchiveFiles(ar_path) :
             if len(ar_part) < 58:
                 continue
             # now read the metadata of the record
-            name = ar_part[-58 : ].split(os.path.sep)[0]
+            name = ar_part[-58:].split(os.path.sep)[0]
         # .lib file format is more complex
         else:
             if ar_part.find(".obj") == -1:
                 continue
             name = ar_part.split(".obj")[-2].split('\x00')[-1].split(os.path.sep)[-1] + ".obj"
             name = name.strip()
         # append the new record
-        if name not in names :
+        if name not in names:
             names.append(name)
     ar_fd.close()
     return names

src/config/anchor.py

@@ -1,9 +1,9 @@
 from anchor_config import *
 from config.utils  import *
 
-def isAnchor(context, seen_strings, seen_consts, functions_list, logger) :
-    """Checks if the given context represents an Anchor function
-    
+def isAnchor(context, seen_strings, seen_consts, functions_list, logger):
+    """Check if the given context represents an Anchor function.
+
     Args:
         context (FunctionContext): canonical representation of a source function
         seen_strings (set): set of unique strings to be used for the scoring
@@ -16,45 +16,45 @@ def isAnchor(context, seen_strings, seen_consts, functions_list, logger) :
     """
     case = 1
     max_case = 5
-    while case <= max_case :
+    while case <= max_case:
         # 1. Huge unique string
-        if case == 1 :
-            huge_strings = filter(lambda x : seen_strings.count(x) == 1, filter(lambda x : len(x) >= STRING_HUGE_LIMIT, context.strings))
-            if len(huge_strings) >= STRING_HUGE_GROUP :
+        if case == 1:
+            huge_strings = filter(lambda x: seen_strings.count(x) == 1, filter(lambda x: len(x) >= STRING_HUGE_LIMIT, context.strings))
+            if len(huge_strings) >= STRING_HUGE_GROUP:
                 logger.debug("Found an Anchor: %s ==> Unique HUGE string (%d)", context.name, len(huge_strings[0]))
                 return True, STRING_HUGE_GROUP, huge_strings
         # 2. Unique string with a function name in it
-        elif case == 2 :
-            for unique_str in filter(lambda x : seen_strings.count(x) == 1, context.strings) :
-                for func_name in functions_list :
-                    if func_name in unique_str :
+        elif case == 2:
+            for unique_str in filter(lambda x: seen_strings.count(x) == 1, context.strings):
+                for func_name in functions_list:
+                    if func_name in unique_str:
                         logger.debug("Found an Anchor: %s ==> Unique string (%s) containing a function name (%s)", context.name, unique_str, func_name)
                         return True, 1, [unique_str]
         # 3. X unique strings with long length
-        elif case == 3 :
-            unique_long_strings = filter(lambda x : seen_strings.count(x) == 1, filter(lambda x : len(x) >= STRING_LONG_LIMIT, context.strings))
-            if len(unique_long_strings) >= STRING_LONG_GROUP :
+        elif case == 3:
+            unique_long_strings = filter(lambda x: seen_strings.count(x) == 1, filter(lambda x: len(x) >= STRING_LONG_LIMIT, context.strings))
+            if len(unique_long_strings) >= STRING_LONG_GROUP:
                 logger.debug("Found an Anchor: %s ==> %d unique long strings", context.name, len(unique_long_strings))
                 return True, STRING_LONG_GROUP, unique_long_strings
         # 4. X unique strings with medium length
-        elif case == 4 :
-            unique_medium_strings = filter(lambda x : seen_strings.count(x) == 1, filter(lambda x : len(x) >= STRING_MEDIUM_LIMIT, context.strings))
-            if len(unique_medium_strings) >= STRING_MEDIUM_GROUP :
+        elif case == 4:
+            unique_medium_strings = filter(lambda x: seen_strings.count(x) == 1, filter(lambda x: len(x) >= STRING_MEDIUM_LIMIT, context.strings))
+            if len(unique_medium_strings) >= STRING_MEDIUM_GROUP:
                 logger.debug("Found an Anchor: %s ==> %d unique medium strings", context.name, len(unique_medium_strings))
                 return True, STRING_MEDIUM_GROUP, unique_medium_strings
         # 5. Unique const with high entropy
-        elif case == 5 :
-            unique_complex_consts = filter(lambda x : seen_consts.count(x) == 1, filter(lambda x : rankConst(x, context) >= CONST_COMPLEX_LIMIT, context.consts))
-            if len(unique_complex_consts) >= CONST_COMPLEX_GROUP :
+        elif case == 5:
+            unique_complex_consts = filter(lambda x: seen_consts.count(x) == 1, filter(lambda x: rankConst(x, context) >= CONST_COMPLEX_LIMIT, context.consts))
+            if len(unique_complex_consts) >= CONST_COMPLEX_GROUP:
                 logger.debug("Found an Anchor: %s ==> %d unique complex consts: %s", context.name, len(unique_complex_consts), hex(unique_complex_consts[0]))
                 return False, CONST_COMPLEX_GROUP, unique_complex_consts
         case += 1
     # we found nothing if we reached this line
     return False, 0, None
 
-def isAgent(context, unique_strings, unique_consts, logger) :
-    """Checks if the given context represents an Agent function inside it's file
-    
+def isAgent(context, unique_strings, unique_consts, logger):
+    """Check if the given context represents an Agent function inside it's file.
+
     Args:
         context (FunctionContext): canonical representation of a source function
         unique_strings (set): set of unique strings to be used for the scoring
@@ -64,26 +64,25 @@ def isAgent(context, unique_strings, unique_consts, logger) :
     Return Value:
         is string criteria (True / False), threshold count, Matching agent criteria (list of string for instance), or None if not an agent
     """
-
     case = 1
     max_case = 3
-    while case <= max_case :
+    while case <= max_case:
         # 1. Medium unique string
-        if case == 1 :
-            medium_strings = filter(lambda x : x in unique_strings, filter(lambda x : len(x) >= STRING_MEDIUM_LIMIT, context.strings))
-            if len(medium_strings) > 0 :
+        if case == 1:
+            medium_strings = filter(lambda x: x in unique_strings, filter(lambda x: len(x) >= STRING_MEDIUM_LIMIT, context.strings))
+            if len(medium_strings) > 0:
                 logger.debug("Found an Agent: %s ==> Unique medium string (%d)", context.name, len(medium_strings[0]))
                 return True, 1, medium_strings
         # 2. X unique strings with short length
-        elif case == 2 :
-            unique_short_strings = filter(lambda x : x in unique_strings, filter(lambda x : len(x) >= STRING_SHORT_LIMIT, context.strings))
-            if len(unique_short_strings) >= STRING_SHORT_GROUP :
+        elif case == 2:
+            unique_short_strings = filter(lambda x: x in unique_strings, filter(lambda x: len(x) >= STRING_SHORT_LIMIT, context.strings))
+            if len(unique_short_strings) >= STRING_SHORT_GROUP:
                 logger.debug("Found an Agent: %s ==> %d unique long strings", context.name, len(unique_short_strings))
                 return True, STRING_SHORT_GROUP, unique_short_strings
         # 3. Unique const with medium entropy
-        elif case == 3 :
-            unique_medium_consts = filter(lambda x : x in unique_consts, filter(lambda x : rankConst(x, context) >= CONST_MEDIUM_LIMIT, context.consts))
-            if len(unique_medium_consts) > 0 :
+        elif case == 3:
+            unique_medium_consts = filter(lambda x: x in unique_consts, filter(lambda x: rankConst(x, context) >= CONST_MEDIUM_LIMIT, context.consts))
+            if len(unique_medium_consts) > 0:
                 logger.debug("Found an Agent: %s ==> %d unique medium consts", context.name, len(unique_medium_consts))
                 return False, 1, unique_medium_consts
         case += 1

src/config/anchor_config.py

@@ -14,7 +14,7 @@
 STRING_SHORT_LIMIT          = 6
 STRING_SHORT_GROUP          = 2
 
-CONST_COMPLEX_LIMIT         = 16 # a bit less than 0xdeadbeef
+CONST_COMPLEX_LIMIT         = 16  # a bit less than 0xdeadbeef
 CONST_COMPLEX_GROUP         = 1
-CONST_MEDIUM_LIMIT          = 9 # a bit less than 0xdead
+CONST_MEDIUM_LIMIT          = 9  # a bit less than 0xdead
 CONST_MEDIUM_GROUP          = 2

src/config/score_config.py

@@ -2,39 +2,39 @@
 ## Scoring Configuration values ##
 ##################################
 
-NUM_BITS_IN_CONST           = 32
-MIN_STR_SIZE                = 4
-CONST_SPECIAL_VALUES        = [0xFFFFFFFF, -1]
-CONST_BOOST_SPECIAL         = 4
-CONST_BOOST_BIT_FLAG        = 6
-CONST_BOOST_SMALL_FUNCS     = 4
-CALL_COUNT_SCORE            = 7
-MATCHED_CALL_SCORE          = 3
-EXTERNAL_COUNT_SCORE        = 5
-STRING_MATCH_SCORE          = 4
-STRING_MISMATCH_SCORE       = 4
-STRING_NAME_SCORE           = 5
-INSTR_COUNT_SCORE           = 0.2
-INSTR_COUNT_THRESHOLD       = 5
-FUNC_FRAME_SCORE            = 0.1
-FRAME_SIZE_THRESHOLD        = 32
-FRAME_SAFETY_GAP            = 16
-BLOCK_MATCH_SCORE           = 0.1
-BLOCK_MISMATCH_SCORE        = 0.1
-FUNC_HINT_SCORE             = 20
-STATIC_VIOLATION_PENALTY    = 20
-LOCATION_BOOST_SCORE        = 15
-LOCATION_BOOST_LOW_THRESHOLD= 0.4
-LOCATION_BOOST_TOP_THRESHOLD= 0.4
-AGENT_BOOST_SCORE           = 20
-EXISTENCE_BOOST_SCORE       = 5
-MINIMAL_BLOCKS_BOOST        = 2
-ARTIFACT_MATCH_SCORE        = 3
-MINIMAL_MATCH_SCORE         = 18.5
-SAFTEY_GAP_SCORE            = 10
-MINIMAL_ISLAND_SCORE        = 0
-MINIMAL_NEIGHBOUR_THRESHOLD = -150
-EXT_FUNC_MATCH_SCORE        = 25
-LIBC_COMP_FUNC_MATCH_SCORE  = 20
-LIBC_FUNC_MATCH_SCORE       = 10
-INSTR_RATIO_COUNT_THRESHOLD = 1
+NUM_BITS_IN_CONST            = 32
+MIN_STR_SIZE                 = 4
+CONST_SPECIAL_VALUES         = [0xFFFFFFFF, -1]
+CONST_BOOST_SPECIAL          = 4
+CONST_BOOST_BIT_FLAG         = 6
+CONST_BOOST_SMALL_FUNCS      = 4
+CALL_COUNT_SCORE             = 7
+MATCHED_CALL_SCORE           = 3
+EXTERNAL_COUNT_SCORE         = 5
+STRING_MATCH_SCORE           = 4
+STRING_MISMATCH_SCORE        = 4
+STRING_NAME_SCORE            = 5
+INSTR_COUNT_SCORE            = 0.2
+INSTR_COUNT_THRESHOLD        = 5
+FUNC_FRAME_SCORE             = 0.1
+FRAME_SIZE_THRESHOLD         = 32
+FRAME_SAFETY_GAP             = 16
+BLOCK_MATCH_SCORE            = 0.1
+BLOCK_MISMATCH_SCORE         = 0.1
+FUNC_HINT_SCORE              = 20
+STATIC_VIOLATION_PENALTY     = 20
+LOCATION_BOOST_SCORE         = 15
+LOCATION_BOOST_LOW_THRESHOLD = 0.4
+LOCATION_BOOST_TOP_THRESHOLD = 0.4
+AGENT_BOOST_SCORE            = 20
+EXISTENCE_BOOST_SCORE        = 5
+MINIMAL_BLOCKS_BOOST         = 2
+ARTIFACT_MATCH_SCORE         = 3
+MINIMAL_MATCH_SCORE          = 18.5
+SAFTEY_GAP_SCORE             = 10
+MINIMAL_ISLAND_SCORE         = 0
+MINIMAL_NEIGHBOUR_THRESHOLD  = -150
+EXT_FUNC_MATCH_SCORE         = 25
+LIBC_COMP_FUNC_MATCH_SCORE   = 20
+LIBC_FUNC_MATCH_SCORE        = 10
+INSTR_RATIO_COUNT_THRESHOLD  = 1