Merge pull request #16 from mkorman90/1.2.0

1.2.0

README.md

@@ -98,7 +98,7 @@ Example output:
 
 ## Recover a registry hive, using transaction logs:
 ```bash
-registry-transaction-logs NTUSER.DAT ntuser.dat.log1 -o recovered_NTUSER.dat 
+registry-transaction-logs NTUSER.DAT -p ntuser.dat.log1 -s ntuser.dat.log2 -o recovered_NTUSER.dat 
 ```
 After recovering, compare the hives with registry-diff to see what changed
 
@@ -118,7 +118,7 @@ for entry in reg.recurse_subkeys(as_json=True):
 
 #### Iterate over a key and get all subkeys and their modification time:
 ```
-for sk in reg.get_key('Software').get_subkeys():
+for sk in reg.get_key('Software').iter_subkeys():
     print(sk.name, convert_wintime(sk.header.last_modified).isoformat())
 
 Adobe 2019-02-03T22:05:32.525965

regipy/cli.py

@@ -148,14 +148,19 @@ def reg_diff(first_hive_path, second_hive_path, output_path, verbose):
 
 @click.command()
 @click.argument('hive_path', type=click.Path(exists=True, dir_okay=False, resolve_path=True), required=True)
-@click.argument('transaction_log_path', type=click.Path(exists=True, dir_okay=False, resolve_path=True), required=True)
+@click.option('-p', 'primary_log_path', type=click.Path(exists=True, dir_okay=False, resolve_path=True), required=True)
+@click.option('-s', 'secondary_log_path', type=click.Path(exists=True, dir_okay=False, resolve_path=True),
+              required=False)
 @click.option('-o', 'output_path', type=click.Path(exists=False, dir_okay=False, resolve_path=True), required=False)
 @click.option('-v', '--verbose', is_flag=True, default=True, help='Verbosity')
-def parse_transaction_log(hive_path, transaction_log_path, output_path, verbose):
+def parse_transaction_log(hive_path, primary_log_path, secondary_log_path, output_path, verbose):
     with logbook.NestedSetup(_get_log_handlers(verbose=verbose)).applicationbound():
-        logger.info(f'Processing hive {hive_path} with transaction log {transaction_log_path}')
+        logger.info(f'Processing hive {hive_path} with transaction log {primary_log_path}')
+        if secondary_log_path:
+            logger.info(f'Processing hive {hive_path} with secondary transaction log {primary_log_path}')
 
-        restored_hive_path, recovered_dirty_pages_count = apply_transaction_logs(hive_path, transaction_log_path,
+        restored_hive_path, recovered_dirty_pages_count = apply_transaction_logs(hive_path, primary_log_path,
+                                                                                 secondary_log_path=secondary_log_path,
                                                                                  restored_hive_path=output_path,
                                                                                  verbose=verbose)
         if recovered_dirty_pages_count:

regipy/plugins/system/active_controlset.py

@@ -1,4 +1,5 @@
 import logbook
+import attr
 
 from regipy.hive_types import SYSTEM_HIVE_TYPE
 from regipy.plugins.plugin import Plugin
@@ -17,6 +18,8 @@ class ActiveControlSetPlugin(Plugin):
     def run(self):
         subkey = self.registry_hive.get_key(SELECT)
         self.entries = [x for x in subkey.iter_values(as_json=self.as_json)]
+        if self.as_json:
+            self.entries = [attr.asdict(x) for x in self.entries]
 
 
 

regipy/plugins/system/computer_name.py

@@ -1,4 +1,5 @@
 import logbook
+import attr
 
 from regipy.exceptions import RegistryValueNotFoundException
 from regipy.hive_types import SYSTEM_HIVE_TYPE
@@ -28,4 +29,3 @@ def run(self):
                 })
             except RegistryValueNotFoundException as ex:
                 continue
-

regipy/plugins/system/timezone_data.py

@@ -1,3 +1,5 @@
+import attr
+
 import logbook
 
 from regipy.hive_types import SYSTEM_HIVE_TYPE
@@ -21,5 +23,9 @@ def run(self):
             tzdata = self.registry_hive.get_key(tzdata_subkey)
             self.entries[tzdata_subkey] = [x for x in tzdata.iter_values(as_json=self.as_json)]
 
+        if self.as_json:
+            for k, v in self.entries.items():
+                self.entries[k] = [attr.asdict(x) for x in v]
+
 
 

regipy/recovery.py

@@ -5,6 +5,7 @@
 
 from construct import Int32ul
 
+from regipy import boomerang_stream
 from regipy.exceptions import RegistryRecoveryException
 from regipy.registry import RegistryHive
 from regipy.structs import TRANSACTION_LOG, REGF_HEADER_SIZE, REGF_HEADER
@@ -28,6 +29,10 @@ def _parse_hvle_block(hive_path, transaction_log_stream, log_size, expected_sequ
 
     while hvle_block_start_offset < log_size:
         logger.info(f'Parsing hvle block at {hvle_block_start_offset}')
+        with boomerang_stream(transaction_log_stream) as x:
+            if x.read(4) != b'HvLE':
+                logger.info('Reached a non HvLE object. stopping')
+                break
 
         parsed_hvle_block = TRANSACTION_LOG.parse_stream(transaction_log_stream)
         logger.info(f'Currently at start of dirty pages: {transaction_log_stream.tell()}')
@@ -42,9 +47,11 @@ def _parse_hvle_block(hive_path, transaction_log_stream, log_size, expected_sequ
             # Write the actual dirty page to the original hive
             target_offset = REGF_HEADER_SIZE + dirty_page_entry.offset
             restored_hive_buffer.seek(target_offset)
+            transaction_log_stream_offset = transaction_log_stream.tell()
             dirty_page_buffer = transaction_log_stream.read(dirty_page_entry.size)
             restored_hive_buffer.write(dirty_page_buffer)
-            logger.info(f'Restored {dirty_page_entry.size} bytes to offset {target_offset}')
+            logger.info(f'Restored {dirty_page_entry.size} bytes to offset {target_offset} '
+                        f'from offset {transaction_log_stream_offset}')
             recovered_dirty_pages_count += 1
 
         # TODO: update hive flags from hvle to original header
@@ -108,17 +115,12 @@ def _parse_dirt_block(hive_path, transaction_log, hbins_data_size):
     return restored_hive_buffer, recovered_dirty_pages_count
 
 
-def apply_transaction_logs(hive_path, transaction_log_path, restored_hive_path=None, verbose=False):
-    if not restored_hive_path:
-        restored_hive_path = f'{hive_path}.restored'
-
-    registry_hive = RegistryHive(hive_path)
+def _parse_transaction_log(registry_hive, hive_path, transaction_log_path):
     log_size = os.path.getsize(transaction_log_path)
-
-    expected_sequence_number = registry_hive.header.secondary_sequence_num
-
     logger.info(f'Log Size: {log_size}')
 
+    expected_sequence_number = registry_hive.header.secondary_sequence_num
+
     with open(transaction_log_path, 'rb') as transaction_log:
 
         transaction_log_regf_header = REGF_HEADER.parse_stream(transaction_log)
@@ -133,10 +135,48 @@ def apply_transaction_logs(hive_path, transaction_log_path, restored_hive_path=N
             hbins_data_size = registry_hive.header.hive_bins_data_size
             restored_hive_buffer, recovered_dirty_pages_count = _parse_dirt_block(hive_path, transaction_log,
                                                                                   hbins_data_size)
+    return restored_hive_buffer, recovered_dirty_pages_count
+
+
+def apply_transaction_logs(hive_path, primary_log_path, secondary_log_path=None,
+                           restored_hive_path=None, verbose=False):
+    """
+    Apply transactions logs
+    :param hive_path: The path to the original hive
+    :param primary_log_path: The path to the primary log path
+    :param secondary_transaction_log_path: The path to the secondary log path (optional).
+    :param restored_hive_path: Path to save the restored hive
+    :param verbose: verbosity
+    :return:
+    """
+    recovered_dirty_pages_total_count = 0
+
+    if not restored_hive_path:
+        restored_hive_path = f'{hive_path}.restored'
+
+    registry_hive = RegistryHive(hive_path)
+
+    log_size = os.path.getsize(primary_log_path)
+    logger.info(f'Log Size: {log_size}')
+
+    restored_hive_buffer, recovered_dirty_pages_count = _parse_transaction_log(registry_hive, hive_path,
+                                                                               primary_log_path)
+
+    recovered_dirty_pages_total_count += recovered_dirty_pages_count
 
     # Write to disk the modified registry hive
     with open(restored_hive_path, 'wb') as f:
         restored_hive_buffer.seek(0)
         f.write(restored_hive_buffer.read())
 
-    return restored_hive_path, recovered_dirty_pages_count
+    if secondary_log_path:
+        registry_hive = RegistryHive(restored_hive_path)
+        restored_hive_buffer, recovered_dirty_pages_count = _parse_transaction_log(registry_hive, restored_hive_path,
+                                                                                   secondary_log_path)
+        # Write to disk the modified registry hive
+        with open(restored_hive_path, 'wb') as f:
+            restored_hive_buffer.seek(0)
+            f.write(restored_hive_buffer.read())
+
+        recovered_dirty_pages_total_count += recovered_dirty_pages_count
+    return restored_hive_path, recovered_dirty_pages_total_count

regipy_tests/conftest.py

@@ -75,3 +75,24 @@ def transaction_log(test_data_dir):
     temp_path = extract_lzma(os.path.join(test_data_dir, 'transactions_ntuser.dat.log1.xz'))
     yield temp_path
     os.remove(temp_path)
+
+
+@pytest.fixture(scope='module')
+def transaction_system(test_data_dir):
+    temp_path = extract_lzma(os.path.join(test_data_dir, 'SYSTEM_B.xz'))
+    yield temp_path
+    os.remove(temp_path)
+
+
+@pytest.fixture(scope='module')
+def transaction_log_1(test_data_dir):
+    temp_path = extract_lzma(os.path.join(test_data_dir, 'SYSTEM_B.LOG1.xz'))
+    yield temp_path
+    os.remove(temp_path)
+
+
+@pytest.fixture(scope='module')
+def transaction_log_2(test_data_dir):
+    temp_path = extract_lzma(os.path.join(test_data_dir, 'SYSTEM_B.LOG2.xz'))
+    yield temp_path
+    os.remove(temp_path)

regipy_tests/profiling.py

@@ -39,11 +39,12 @@ def get_file_from_tests(file_name):
     os.remove(tempfile_path)
 
 
-registry_path = 'SAM.xz'
-logger.info(f'Iterating over all subkeys in {registry_path}')
+registry_path = 'SYSTEM.xz'
+print(f'Iterating over all subkeys in {registry_path}')
 with profiling():
     with get_file_from_tests(registry_path) as reg:
         registry_hive = RegistryHive(reg)
         keys = [x for x in registry_hive.recurse_subkeys()]
+print(f'Done.')
 
 

regipy_tests/tests.py

@@ -179,6 +179,20 @@ def test_ntuser_apply_transaction_logs(transaction_ntuser, transaction_log):
     assert len([x for x in found_differences if x[0] == 'new_value']) == 59
 
 
+def test_system_apply_transaction_logs(transaction_system, transaction_log_1, transaction_log_2):
+    output_path = os.path.join(mkdtemp(), 'recovered_hive.dat')
+    restored_hive_path, recovered_dirty_pages_count = apply_transaction_logs(transaction_system,
+                                                                             primary_log_path=transaction_log_1,
+                                                                             secondary_log_path=transaction_log_2,
+                                                                             restored_hive_path=output_path)
+    assert recovered_dirty_pages_count == 315
+
+    found_differences = compare_hives(transaction_system, restored_hive_path)
+    assert len(found_differences) == 2486
+    assert len([x for x in found_differences if x[0] == 'new_subkey']) == 2472
+    assert len([x for x in found_differences if x[0] == 'new_value']) == 13
+
+
 def test_hive_serialization(ntuser_hive, temp_output_file):
     registry_hive = RegistryHive(ntuser_hive)
     registry_hive.dump_hive_to_json(temp_output_file, registry_hive.root, verbose=False)

setup.py

@@ -15,7 +15,7 @@
 def main():
     setup(name='regipy',
           packages=find_packages(),
-          version='1.1.4',
+          version='1.2.0',
           description='Python Registry Parser',
           long_description=readme,
           author='Martin G. Korman',