Merge v0.9.8 changes from internal repo

requirements.txt

@@ -1,4 +1,6 @@
 capstone>=4.0.1
 nose>=1.3.7
+packaging>=20.9
+py-tlsh>=4.5.0
 pyelftools>=0.25
 python-tlsh>=3.17.0

telfhash/VERSION

@@ -1 +1 @@
-0.9.6
+0.9.8

telfhash/grouping.py

@@ -32,10 +32,10 @@
 def get_combination(telfhash_data):
 
     #
-    # TLSH hash is 70 characters long. if the telfhash is not 70
+    # TLSH hash is 72 characters long. if the telfhash is not 72
     # characters in length, exclude from the list
     #
-    files_list = [x for x in list(telfhash_data.keys()) if telfhash_data[x]["telfhash"] is not None and len(telfhash_data[x]["telfhash"]) == 70]
+    files_list = [x for x in list(telfhash_data.keys()) if telfhash_data[x]["telfhash"] is not None and len(telfhash_data[x]["telfhash"]) == 72]
 
     #
     # get the combination of all the possible pairs of filenames
@@ -67,7 +67,7 @@ def get_distances(telfhash_data, files_combination):
     for element in files_combination:
         file1 = element[0]
         file2 = element[1]
-        distance = tlsh.diff(telfhash_data[file1]["telfhash"], telfhash_data[file2]["telfhash"])
+        distance = tlsh.diff(telfhash_data[file1]["telfhash"].upper(), telfhash_data[file2]["telfhash"].upper())
 
         distances.append((file1, file2, distance))
 

telfhash/telfhash.py

@@ -46,6 +46,14 @@
 with open(os.path.join(HERE, "VERSION")) as version_file:
     VERSION = version_file.read().strip()
 
+
+# we will only support the new TLSH versions
+#
+TLSH_MIN_VERSION = '4.2.1'
+from packaging import version
+if version.parse(tlsh.__version__) < version.parse(TLSH_MIN_VERSION):
+    raise Exception("telfhash v{} only supports tlsh version {} and higher. You have tlsh version {}".format(VERSION, TLSH_MIN_VERSION, tlsh.__version__))
+
 EXCLUSIONS_REGEX = [
     r"^[_\.].*$",       # Function names starting with . or _
     "^.*64$",           # x64-64 specific functions