Transition to Python3

.gitignore

@@ -1,12 +1,14 @@
 
 .vscode/
+.idea/
 
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[co]
 *_test.py
 mac_apt_compiled*
 mac_apt_singleplugin_compiled*
+mac_apt_artifact_only_compiled*
 mac_apt_vmdk*
 *$py.class
 

Libraries_For_Windows/Readme.txt

@@ -0,0 +1,32 @@
+You will need to copy the .pyd files contained here into your python 
+installation folder under Lib\site-packages\
+
+This is specific for Python 3.7 and will not work with other versions. It is
+also specific to your platform (32/64 bits). 
+
+The 32-bit versions are located in the 32bit_py37 folder. 
+The 64-bit versions are located in the 64bit_py37 folder. 
+
+Copy only the files for your platform into Lib\site-packages\
+
+If you are unsure about your python platform (32/64), then run python.exe and 
+run the following 2 lines of code:
+
+ import sys
+ sys.maxsize > 2**32
+
+If the reply is True, you are on 64 bit python, else 32 bit.
+
+The .pyd files are DLLs that have been compiled using Visual Studio 2017. 
+These files are dependencies that need to be installed for mac_apt to run on
+Windows (when running from code). 
+
+pyewf.pyd  -> https://github.com/libyal/libewf
+pytsk3.pyd -> https://github.com/py4n6/pytsk
+pyvmdk.pyd -> https://github.com/libyal/libvmdk
+lzfse.pyf  -> https://github.com/ydkhatri/pylzfse
+
+lzfse.pyd is a fork of the pylzfse project (https://github.com/dimkr/pylzfse) with 
+a few modifications to work with the compression parameters used in macOS. That 
+project in turn depends on Apple's reference implementation of lzvn/lzfse available
+at https://github.com/lzfse/lzfse.

README.md

@@ -3,29 +3,32 @@ macOS Artifact Parsing Tool
 
 mac_apt is a DFIR tool to process Mac computer full disk images (or live machines) and extract data/metadata useful for forensic investigation. It is a python based framework, which has plugins to process individual artifacts (such as Safari internet history, Network interfaces, Recently accessed files & volumes, ..)
 
-#### Project Status: alpha, experimental
-#### Requirements: 32 bit Python 2.7
+#### Project Status: WIP
+#### Requirements: 32/64 bit Python 3.7
 
 #### Features:
 * Cross platform (no dependency on pyobjc)
-* Works on E01, DD, split-DD, DMG (no compression) & mounted images (good for nix, limited support on windows)
+* Works on E01, VMDK, DD, split-DD, DMG (no compression) & mounted images (good for nix, limited support on windows)
 * XLSX, CSV, Sqlite outputs
 * Analyzed files/artifacts are exported for later review
 * zlib, lzvn, lzfse compressed files are supported!
 * APFS is now supported, you can process HighSierra images now!
 
 Available Plugins (artifacts parsed) | Description 
 ------------------ | ---------------
+AUTOSTART | Retrieves programs, daemons, services set to start at boot/login
 BASHSESSIONS | Reads bash (Terminal) sessions & history for every user
 BASICINFO | Basic machine & OS configuration like SN, timezone, computer name, last logged in user, HFS info
 BLUETOOTH | Gets Bluetooth Artifacts
 DOCKITEMS | Reads the Dock plist for every user
 DOMAINS | Active Directory Domain(s) that the mac is connected to
 FSEVENTS | Reads file system event logs (from .fseventsd)
 IDEVICEBACKUPS | Reads and exports iPhone/iPad backup databases
+IDEVICEINFO | Reads and exports connected iDevice details
 IMESSAGE | Read iMessage chats
 INETACCOUNTS | Retrieve configured internet accounts (iCloud, Google, Linkedin, facebook..)
 INSTALLHISTORY | Software Installation History
+MSOFFICE | Reads Word, Excel, Powerpoint and other office MRU/accessed file paths
 NETUSAGE | Read network usage data statistics per application
 NETWORKING | Interfaces, last IP address, MAC address, DHCP ..
 NOTES | Reads notes databases
@@ -36,15 +39,16 @@ RECENTITEMS | Recently accessed Servers, Documents, Hosts, Volumes & Application
 SAFARI | Internet history, downloaded file information, cookies and more from Safari caches
 SPOTLIGHT | Reads the spotlight index databases
 SPOTLIGHTSHORTCUTS | User typed data in the spotlight bar & targeted document/app
+UNIFIEDLOGS | Reads macOS unified logging logs from .tracev3 files
 USERS | Local & Domain user information - name, UID, UUID, GID, account creation & password set dates, pass hints, homedir & Darwin paths
 WIFI | Gets wifi network information
 
 ### Coming soon..
 * More plugins
 * More documentation
-* Python 3 version
 
-For installation and other information, see https://github.com/ydkhatri/mac_apt/wiki
+For installation (to run from code) see https://github.com/ydkhatri/mac_apt/wiki/Installation-for-Python3.7  
+Other information/documentation, see https://github.com/ydkhatri/mac_apt/wiki
 
 To download, proceed here - https://github.com/ydkhatri/mac_apt/releases
 

mac_apt.py

@@ -13,29 +13,28 @@
    For usage information, run: 
      python mac_apt.py -h
 
-   NOTE: This currently works only on Python2.
+   NOTE: This currently works only on Python3.7 or higher.
 
 '''
-from __future__ import print_function
-from __future__ import unicode_literals
 
-import sys
-import os
 import argparse
-import binascii
+import logging
+import os
 import pyewf
 import pytsk3
+import pyvmdk
+import sys
+import textwrap
+import time
 import traceback
+from uuid import UUID
 import plugins.helpers.macinfo as macinfo
 from plugins.helpers.apfs_reader import ApfsContainer, ApfsDbInfo
 from plugins.helpers.writer import *
 from plugins.helpers.disk_report import *
-import logging
-import time
-import textwrap
 from plugin import *
 
-__VERSION = "0.3"
+__VERSION = "0.4"
 __PROGRAMNAME = "macOS Artifact Parsing Tool"
 __EMAIL = "yogesh@swiftforensics.com"
 
@@ -44,13 +43,13 @@ def IsItemPresentInList(collection, item):
     try:
         collection.index(item)
         return True
-    except Exception:
+    except ValueError:
         pass
     return False
 
 def CheckInputType(input_type):
     input_type = input_type.upper()
-    return input_type in ['E01','DD','MOUNTED']
+    return input_type in ['E01','DD','VMDK','MOUNTED']
 
 ######### FOR HANDLING E01 file ###############
 class ewf_Img_Info(pytsk3.Img_Info):
@@ -89,6 +88,63 @@ def GetImgInfoObjectForE01(path):
 
 ####### End special handling for E01 #########
 
+######### FOR HANDLING VMDK file ###############
+class vmdk_Img_Info(pytsk3.Img_Info):
+  def __init__(self, vmdk_handle):
+    self._vmdk_handle = vmdk_handle
+    super(vmdk_Img_Info, self).__init__(
+        url="", type=pytsk3.TSK_IMG_TYPE_EXTERNAL)
+
+  def close(self):
+    self._vmdk_handle.close()
+
+  def read(self, offset, size):
+    self._vmdk_handle.seek(offset)
+    return self._vmdk_handle.read(size)
+
+  def get_size(self):
+    return self._vmdk_handle.get_media_size()
+
+def OpenExtentDataFiles(vmdk_handle, base_directory):
+    '''Because vmdk_handle.open_extent_data_files() is broken in 20170226'''
+    extent_data_files = []
+    for extent_descriptor in vmdk_handle.extent_descriptors:
+        extent_data_filename = extent_descriptor.filename
+
+        _, path_separator, filename = extent_data_filename.rpartition("/")
+        if not path_separator:
+            _, path_separator, filename = extent_data_filename.rpartition("\\")
+
+        if not path_separator:
+            filename = extent_data_filename
+
+        extent_data_file_path = os.path.join(base_directory, filename)
+
+        if not os.path.exists(extent_data_file_path):
+            break
+
+        extent_data_files.append(extent_data_file_path)
+
+    if len(extent_data_files) != vmdk_handle.number_of_extents:
+        raise RuntimeError("Unable to locate all extent data files.")
+
+    file_objects = []
+    for extent_data_file_path in extent_data_files:
+        file_object = open(extent_data_file_path, "rb")
+        file_objects.append(file_object)
+
+    vmdk_handle.open_extent_data_files_file_objects(file_objects)
+
+def GetImgInfoObjectForVMDK(path):
+    vmdk_handle = pyvmdk.handle()
+    vmdk_handle.open(path)
+    base_directory = os.path.dirname(path)
+    #vmdk_handle.open_extent_data_files() Broken in current version #20170226
+    OpenExtentDataFiles(vmdk_handle, base_directory)
+    img_info = vmdk_Img_Info(vmdk_handle)
+    return img_info
+####### End special handling for VMDK #########
+
 def FindOsxFiles(mac_info):
     if mac_info.IsValidFilePath('/System/Library/CoreServices/SystemVersion.plist'):
         if mac_info.IsValidFilePath("/System/Library/Kernels/kernel") or \
@@ -123,8 +179,7 @@ def IsOsxPartition(img, partition_start_offset, mac_info):
             log.error ("Could not open / (root folder on partition)")
             log.debug ("Exception info", exc_info=True)
     except Exception as ex:
-        log.info(" Error: Failed to detect/parse file system!" + str(ex))
-        log.exception("Exception") #traceback.print_exc(
+        log.exception("Exception")
     return False
 
 def IsApfsContainer(img, partition_start_offset):
@@ -133,12 +188,14 @@ def IsApfsContainer(img, partition_start_offset):
         if img.read(partition_start_offset + 0x20, 4) == b'NXSB':
             return True
     except:
-        raise Exception('Cannot seek into image @ offset {}'.format(partition_start_offset + 0x20))
+        raise ValueError('Cannot seek into image @ offset {}'.format(partition_start_offset + 0x20))
     return False
 
 def GetApfsContainerUuid(img, container_start_offset):
-    uuid = img.read(container_start_offset + 72, 16)
-    return binascii.hexlify(uuid).upper()
+    '''Returns a UUID object'''
+    uuid_bytes = img.read(container_start_offset + 72, 16)
+    uuid = UUID(bytes=uuid_bytes)
+    return uuid
 
 def FindOsxPartitionInApfsContainer(img, vol_info, container_size, container_start_offset, container_uuid):
     global mac_info
@@ -151,7 +208,7 @@ def FindOsxPartitionInApfsContainer(img, vol_info, container_size, container_sta
     try:
         # start db
         use_existing_db = False
-        apfs_sqlite_path = os.path.join(mac_info.output_params.output_path, "APFS_Volumes_" + container_uuid + ".db")
+        apfs_sqlite_path = os.path.join(mac_info.output_params.output_path, "APFS_Volumes_" + str(container_uuid).upper() + ".db")
         if os.path.exists(apfs_sqlite_path): # Check if db already exists
             existing_db = SqliteWriter()     # open & check if it has the correct data
             existing_db.OpenSqliteDb(apfs_sqlite_path)
@@ -164,6 +221,8 @@ def FindOsxPartitionInApfsContainer(img, vol_info, container_size, container_sta
             else:
                 # db does not seem up to date, create a new one and read info
                 existing_db.CloseDb()
+                log.info('Found an existing APFS_Volumes.db in the output folder, but it is STALE, creating a new one!')
+                os.remove(apfs_sqlite_path)
         if not use_existing_db:
             apfs_sqlite_path = SqliteWriter.CreateSqliteDb(apfs_sqlite_path) # Will create with next avail file name
             mac_info.apfs_db = SqliteWriter()
@@ -184,6 +243,7 @@ def FindOsxPartitionInApfsContainer(img, vol_info, container_size, container_sta
                 continue
             if vol.is_encrypted: continue
             mac_info.osx_FS = vol
+            vol.dbo = mac_info.apfs_db
             if FindOsxFiles(mac_info):
                 return True
         # Did not find macOS installation
@@ -209,7 +269,7 @@ def FindOsxPartition(img, vol_info, vs_info):
 
             if IsApfsContainer(img, partition_start_offset):
                 uuid = GetApfsContainerUuid(img, partition_start_offset)
-                log.info('Found an APFS container with uuid: {}-{}-{}-{}-{}'.format(uuid[0:8], uuid[8:12], uuid[12:16], uuid[16:20], uuid[20:]))
+                log.info('Found an APFS container with uuid: {}'.format(str(uuid).upper()))
                 return FindOsxPartitionInApfsContainer(img, vol_info, vs_info.block_size * part.len, partition_start_offset, uuid)
 
             elif IsOsxPartition(img, partition_start_offset, mac_info): # Assumes there is only one single OSX installation partition
@@ -259,7 +319,7 @@ def SetupExportLogger(output_params):
 arg_parser = argparse.ArgumentParser(description='mac_apt is a framework to process forensic artifacts on a Mac OSX system\n'\
                                                  'You are running {} version {}'.format(__PROGRAMNAME, __VERSION),
                                     epilog=plugins_info, formatter_class=argparse.RawTextHelpFormatter)
-arg_parser.add_argument('input_type', help='Specify Input type as either E01, DD or MOUNTED')
+arg_parser.add_argument('input_type', help='Specify Input type as either E01, DD, VMDK or MOUNTED')
 arg_parser.add_argument('input_path', help='Path to OSX image/volume')
 arg_parser.add_argument('-o', '--output_path', help='Path where output files will be created')
 arg_parser.add_argument('-x', '--xlsx', action="store_true", help='Save output in excel spreadsheet(s)')
@@ -342,6 +402,9 @@ def SetupExportLogger(output_params):
     if args.input_type.upper() == 'E01':
         img = GetImgInfoObjectForE01(args.input_path) # Use this function instead of pytsk3.Img_Info()
         mac_info = macinfo.MacInfo(output_params)
+    elif args.input_type.upper() == 'VMDK':
+        img = GetImgInfoObjectForVMDK(args.input_path) # Use this function instead of pytsk3.Img_Info()
+        mac_info = macinfo.MacInfo(output_params)
     elif args.input_type.upper() == 'DD':
         img = pytsk3.Img_Info(args.input_path) # Works for split dd images too! Works for DMG too, if no compression/encryption is used!
         mac_info = macinfo.MacInfo(output_params)
@@ -370,7 +433,7 @@ def SetupExportLogger(output_params):
             log.info(" Info: Probably not a disk image, trying to parse as a File system")
             if IsApfsContainer(img, 0):
                 uuid = GetApfsContainerUuid(img, 0)
-                log.info('Found an APFS container with uuid: {}-{}-{}-{}-{}'.format(uuid[0:8], uuid[8:12], uuid[12:16], uuid[16:20], uuid[20:]))
+                log.info('Found an APFS container with uuid: {}'.format(str(uuid).upper()))
                 found_osx = FindOsxPartitionInApfsContainer(img, None, img.get_size(), 0, uuid)
             else:
                 found_osx = IsOsxPartition(img, 0, mac_info)
@@ -392,7 +455,7 @@ def SetupExportLogger(output_params):
             except Exception as ex:
                 log.exception ("An exception occurred while running plugin - {}".format(plugin.__Plugin_Name))
 else:
-    log.warning (":( Could not find a partition having an OSX installation on it")
+    log.warning (":( Could not find a partition having a macOS installation on it")
 
 log.info("-"*50)