Fix PDF decryption for 256-bit AES with V=5

CHANGELOG.md

@@ -100,3 +100,4 @@ This is v10.0.9, a release candidate for v10.1.0. I don't expect there to be maj
 ## Fixes on master (not yet released):
 
 - Fix a bug where decrypting a 40-bit RC4 pdf with R=2 didn't work.
+- Fix a bug where decrypting a 256-bit AES pdf with V=5 didn't work.

DeDRM_plugin/ineptpdf.py

@@ -1366,8 +1366,7 @@ def recover_encryption_key_with_password(self, password, docid, param):
 
     def process_with_aes(self, key, encrypt, data, repetitions = 1, iv = None):
         if iv is None:
-            keylen = len(key)
-            iv = bytes([0x00]*keylen)
+            iv = bytes(bytearray(16))
 
         aes = AES.new(key, AES.MODE_CBC, iv)
 
@@ -1395,10 +1394,18 @@ def hash_V5(self, password, salt, userdata, param):
                     raise Exception("K1 < 32 ...")
                 #def process_with_aes(self, key: bytes, encrypt: bool, data: bytes, repetitions: int = 1, iv: bytes = None):
                 E = self.process_with_aes(K[:16], True, K1, 64, K[16:32])
-                K = (hashlib.sha256, hashlib.sha384, hashlib.sha512)[sum(E) % 3](E).digest()
+                E = bytearray(E)
+
+                E_mod_3 = 0
+                for i in range(16):
+                    E_mod_3 += E[i]
+
+                E_mod_3 %= 3
+
+                K = (hashlib.sha256, hashlib.sha384, hashlib.sha512)[E_mod_3](E).digest()
 
                 if round_number >= 64:
-                    ch = int.from_bytes(E[-1:], "big", signed=False)
+                    ch = E[-1:][0]  # get last byte
                     if ch <= round_number - 32:
                         done = True