From 672c7f6b1583781b7cbd3d19653ad6c33be716fc Mon Sep 17 00:00:00 2001 From: K0lb3 Date: Sat, 28 Jan 2023 22:07:47 +0100 Subject: [PATCH 01/14] MeshRenderExporter - use m_Name instead of name --- UnityPy/export/MeshRendererExporter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/UnityPy/export/MeshRendererExporter.py b/UnityPy/export/MeshRendererExporter.py index 83fe11db..89992b22 100644 --- a/UnityPy/export/MeshRendererExporter.py +++ b/UnityPy/export/MeshRendererExporter.py @@ -114,7 +114,7 @@ def clt(color): # color to tuple if not texEnv.m_Texture: continue tex = texEnv.m_Texture.read() - texName = f"{tex.name if tex.name else key}.png" + texName = f"{tex.m_Name if tex.m_Name else key}.png" if key == "_MainTex": sb.append(f"map_Kd {texName}") elif key == "_BumpMap": From ea537e1e58c477aa2ac64a3e228308b00fa27d16 Mon Sep 17 00:00:00 2001 From: K0lb3 Date: Sat, 28 Jan 2023 22:09:19 +0100 Subject: [PATCH 02/14] Texture2DConverter - image - error if there is no image data --- UnityPy/export/Texture2DConverter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/UnityPy/export/Texture2DConverter.py b/UnityPy/export/Texture2DConverter.py index 6edb9a8a..b757a3c7 100644 --- a/UnityPy/export/Texture2DConverter.py +++ b/UnityPy/export/Texture2DConverter.py @@ -100,7 +100,7 @@ def get_image_from_texture2d(texture_2d, flip=True) -> Image.Image: """ image_data = copy(bytes(texture_2d.image_data)) if not image_data: - return Image.new("RGB", (0, 0)) + raise ValueError("Texture2D has no image data") texture_format = ( texture_2d.m_TextureFormat From 0b616fd29da842e21d0d6c1444f3ff546b34df5c Mon Sep 17 00:00:00 2001 From: K0lb3 Date: Sat, 28 Jan 2023 22:09:44 +0100 Subject: [PATCH 03/14] Texture2D - load image data from resource lazily --- UnityPy/classes/Texture2D.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/UnityPy/classes/Texture2D.py b/UnityPy/classes/Texture2D.py index 5c5539a3..705aeaba 100644 --- a/UnityPy/classes/Texture2D.py +++ b/UnityPy/classes/Texture2D.py @@ -47,6 +47,13 @@ def image(self, img): @property def image_data(self): + if not self._image_data and self.m_StreamData is not None: + self._image_data = get_resource_data( + self.m_StreamData.path, + self.assets_file, + self.m_StreamData.offset, + self.m_StreamData.size, + ) return self._image_data def reset_streamdata(self): @@ -166,13 +173,8 @@ def __init__(self, reader): if version >= (5, 3): # 5.3 and up # always read the StreamingInfo for resaving self.m_StreamData = StreamingInfo(reader, version) - if image_data_size == 0 and self.m_StreamData.path: - self._image_data = get_resource_data( - self.m_StreamData.path, - self.assets_file, - self.m_StreamData.offset, - self.m_StreamData.size, - ) + # don't read the data directly, + # as we don't want the parser break if the file is missing def save(self, writer: EndianBinaryWriter = None): if writer is None: From 0ceed24eaeb764afa12338cd133162df63efd859 Mon Sep 17 00:00:00 2001 From: K0lb3 Date: Sat, 28 Jan 2023 22:11:28 +0100 Subject: [PATCH 04/14] rework dependency loading and resolving --- UnityPy/classes/PPtr.py | 34 +++------ UnityPy/environment.py | 110 +++++++++++++++++++++--------- UnityPy/files/BundleFile.py | 6 +- UnityPy/files/File.py | 29 +++----- UnityPy/helpers/ImportHelper.py | 24 ++++++- UnityPy/helpers/ResourceReader.py | 80 ++++++++-------------- 6 files changed, 152 insertions(+), 131 deletions(-) diff --git a/UnityPy/classes/PPtr.py b/UnityPy/classes/PPtr.py index cae164a3..96ae6d64 100644 --- a/UnityPy/classes/PPtr.py +++ b/UnityPy/classes/PPtr.py @@ -1,10 +1,6 @@ from ..files import ObjectReader from ..streams import EndianBinaryWriter -from ..helpers import ImportHelper -from .. import files -from ..enums import FileType, ClassIDType -import os -from .. import environment +from ..enums import ClassIDType def save_ptr(obj, writer: EndianBinaryWriter): @@ -33,7 +29,9 @@ def save(self, writer: EndianBinaryWriter): def get_obj(self): if self._obj != None: return self._obj + manager = None + if self.file_id == 0: manager = self.assets_file @@ -43,27 +41,13 @@ def get_obj(self): external_name = self.external_name # try to find it in the already registered cabs manager = environment.get_cab(external_name) - + # not found, load all dependencies and try again if not manager: - # guess we have to try to find it as file then - path = environment.path - if path is not None: - basename = os.path.basename(external_name) - possible_names = [basename, basename.lower(), basename.upper()] - for root, dirs, files in os.walk(path): - for name in files: - if name in possible_names: - manager = environment.load_file( - os.path.join(root, name) - ) - environment.register_cab(name, manager) - break - else: - # else is reached if the previous loop didn't break - continue - break - if manager and self.path_id in manager.objects: - self._obj = manager.objects[self.path_id] + self.assets_file.load_dependencies([external_name]) + manager = environment.get_cab(external_name) + + if manager is not None: + self._obj = manager.objects.get(self.path_id) else: self._obj = None if self.external_name: diff --git a/UnityPy/environment.py b/UnityPy/environment.py index c87fcbd6..093e0eed 100644 --- a/UnityPy/environment.py +++ b/UnityPy/environment.py @@ -1,6 +1,7 @@ from typing import List, Callable, Dict, Union import io import os +import ntpath from zipfile import ZipFile import re from . import files @@ -17,12 +18,16 @@ class Environment: files: dict cabs: dict path: str + local_files: List[str] + local_files_simple: List[str] def __init__(self, *args): self.files = {} self.cabs = {} self.path = None self.out_path = os.path.join(os.getcwd(), "output") + self.local_files = [] + self.local_files_simple = [] if args: for arg in args: @@ -78,6 +83,7 @@ def load_file( file: Union[io.IOBase, str], parent: Union["Environment", File] = None, name: str = None, + is_dependency: bool = False, ): if not parent: parent = self @@ -101,37 +107,28 @@ def load_file( typ, reader = ImportHelper.check_file_type(file) - try: - stream_name = ( - name - if name - else getattr( - file, - "name", - str(file.__hash__()) if hasattr(file, "__hash__") else "", - ) + stream_name = ( + name + if name + else getattr( + file, + "name", + str(file.__hash__()) if hasattr(file, "__hash__") else "", ) + ) + + if typ == FileType.ZIP: + f = self.load_zip_file(file) + else: + f = ImportHelper.parse_file( + reader, self, name=stream_name, typ=typ, is_dependency=is_dependency + ) + + if isinstance(f, (SerializedFile, EndianBinaryReader)): + self.register_cab(stream_name, f) - if typ == FileType.AssetsFile: - f = files.SerializedFile(reader, parent, name=stream_name) - self.register_cab(stream_name, f) - elif typ == FileType.BundleFile: - f = files.BundleFile(reader, parent, name=stream_name) - elif typ == FileType.WebFile: - f = files.WebFile(reader, parent, name=stream_name) - elif typ == FileType.ZIP: - f = self.load_zip_file(file) - elif typ == FileType.ResourceFile: - f = EndianBinaryReader(file) - self.register_cab(stream_name, f) - - self.files[stream_name] = f - return f - except Exception as e: - # just to be sure - # cuz the SerializedFile detection isn't perfect - print("Error loading, reverting to EndianBinaryReader:\n", str(e)) - return EndianBinaryReader(file) + self.files[stream_name] = f + return f def load_zip_file(self, value): buffer = None @@ -166,6 +163,8 @@ def search(item): ret = [] if not isinstance(item, Environment) and getattr(item, "objects", None): # serialized file + if getattr(item, "is_dependency", False): + return [] return [val for val in item.objects.values()] elif getattr(item, "files", None): # WebBundle and BundleFile @@ -184,7 +183,7 @@ def container(self) -> Dict[str, ObjectReader]: return { path: obj for f in self.files.values() - if isinstance(f, File) + if isinstance(f, File) and not f.is_dependency for path, obj in f.container.items() } @@ -196,6 +195,8 @@ def assets(self) -> list: def gen_all_asset_files(file, ret=[]): for f in getattr(file, "files", {}).values(): + if getattr(f, "is_dependency", False): + continue if isinstance(f, SerializedFile): ret.append(f) else: @@ -218,7 +219,7 @@ def register_cab(self, name: str, item: File) -> None: item : File The file to register. """ - self.cabs[os.path.basename(name.lower())] = item + self.cabs[simplify_name(name)] = item def get_cab(self, name: str) -> File: """ @@ -234,7 +235,7 @@ def get_cab(self, name: str) -> File: File The cab file. """ - return self.cabs.get(os.path.basename(name.lower()), None) + return self.cabs.get(simplify_name(name), None) def load_assets(self, assets: List[str], open_f: Callable[[str], io.IOBase]): """ @@ -271,3 +272,48 @@ def load_assets(self, assets: List[str], open_f: Callable[[str], io.IOBase]): else: data = open_f(path).read() self.load_file(data, name=path) + + def find_file(self, name: str, is_dependency: bool = True) -> Union[File, None]: + """ + Finds a file in the environment. + + Parameters + ---------- + name : str + The name of the file. + is_dependency : bool + Whether the file is a dependency. + + Returns + ------- + File | None + The file if it was found, otherwise None. + """ + simple_name = simplify_name(name) + cab = self.get_cab(simple_name) + if cab: + return cab + + if len(self.local_files) == 0 and self.path: + for root, _, files in os.walk(self.path): + for name in files: + self.local_files.append(os.path.join(root, name)) + + if name in self.local_files: + fp = name + elif simple_name in self.local_files_simple: + fp = self.local_files[self.local_files_simple.index(simple_name)] + else: + raise FileNotFoundError(f"File {name} not found in {self.path}") + + f = self.load_file(fp, name=name, is_dependency=is_dependency) + return f + + +def simplify_name(name: str) -> str: + """Simplifies a name by: + - removing the extension + - removing the path + - converting to lowercase + """ + return ntpath.basename(name).lower() diff --git a/UnityPy/files/BundleFile.py b/UnityPy/files/BundleFile.py index 618e9753..5340434a 100644 --- a/UnityPy/files/BundleFile.py +++ b/UnityPy/files/BundleFile.py @@ -24,8 +24,10 @@ class BundleFile(File.File): dataflags: Tuple[ArchiveFlags, ArchiveFlagsOld] decryptor: ArchiveStorageManager.ArchiveStorageDecryptor = None - def __init__(self, reader: EndianBinaryReader, parent: File, name: str = None): - super().__init__(parent=parent, name=name) + def __init__( + self, reader: EndianBinaryReader, parent: File, name: str = None, **kwargs + ): + super().__init__(parent=parent, name=name, **kwargs) signature = self.signature = reader.read_string_to_null() self.version = reader.read_u_int() self.version_player = reader.read_string_to_null() diff --git a/UnityPy/files/File.py b/UnityPy/files/File.py index 4d3f43d5..d42d4460 100644 --- a/UnityPy/files/File.py +++ b/UnityPy/files/File.py @@ -11,21 +11,26 @@ class File(object): name: str files: dict + environment: "Environment" cab_file: str is_changed: bool signature: str packer: str + is_dependency: bool # parent: File # environment: Environment - def __init__(self, parent=None, name=None): + def __init__(self, parent=None, name: str = None, is_dependency: bool = False): self.files = {} self.is_changed = False self.cab_file = "CAB-UnityPy_Mod.resS" self.parent = parent - self.environment = self.environment = getattr(parent, "environment", parent) if parent else None + self.environment = self.environment = ( + getattr(parent, "environment", parent) if parent else None + ) self.name = basename(name) if isinstance(name, str) else "" + self.is_dependency = is_dependency def get_assets(self): if isinstance(self, SerializedFile.SerializedFile): @@ -67,23 +72,12 @@ def read_files(self, reader: EndianBinaryReader, files: list): for node in files: reader.Position = node.offset name = node.path - f = EndianBinaryReader( + reader = EndianBinaryReader( reader.read(node.size), offset=(reader.BaseOffset + node.offset) ) - # f._flag = getattr(node, "flags", None) # required for save - typ, _ = ImportHelper.check_file_type(f) - if typ == FileType.BundleFile: - f = BundleFile.BundleFile(f, self, name=name) - elif typ == FileType.WebFile: - f = WebFile.WebFile(f, self, name=name) - elif typ == FileType.AssetsFile: - # pre-check if resource file - if not name.endswith((".resS", ".resource", ".config", ".xml", ".dat")): - # try to load the file as serialized file - try: - f = SerializedFile.SerializedFile(f, self, name=name) - except ValueError: - pass + f = ImportHelper.parse_file( + reader, self.parent, name, is_dependency=self.is_dependency + ) if isinstance(f, (EndianBinaryReader, SerializedFile.SerializedFile)): if self.environment: @@ -161,4 +155,3 @@ def mark_changed(self): # recursive import requires the import down here from . import BundleFile, SerializedFile, WebFile, ObjectReader - diff --git a/UnityPy/helpers/ImportHelper.py b/UnityPy/helpers/ImportHelper.py index 39f7365e..53599ef5 100644 --- a/UnityPy/helpers/ImportHelper.py +++ b/UnityPy/helpers/ImportHelper.py @@ -3,6 +3,7 @@ from .CompressionHelper import BROTLI_MAGIC, GZIP_MAGIC from ..enums import FileType from ..streams import EndianBinaryReader +from .. import files def file_name_without_extension(file_name: str) -> str: @@ -53,7 +54,6 @@ def check_file_type(input_) -> Union[FileType, EndianBinaryReader]: if reader.Length < 20: return FileType.ResourceFile, reader - signature = reader.read_string_to_null(20) reader.Position = 0 @@ -119,3 +119,25 @@ def check_file_type(input_) -> Union[FileType, EndianBinaryReader]: return FileType.ResourceFile, reader else: return FileType.AssetsFile, reader + + +def parse_file( + reader: EndianBinaryReader, + parent, + name: str, + typ: FileType = None, + is_dependency=False, +): + if typ is None: + typ, _ = check_file_type(reader) + if typ == FileType.AssetsFile and not name.endswith( + (".resS", ".resource", ".config", ".xml", ".dat") + ): + f = files.SerializedFile(reader, parent, name=name, is_dependency=is_dependency) + elif typ == FileType.BundleFile: + f = files.BundleFile(reader, parent, name=name, is_dependency=is_dependency) + elif typ == FileType.WebFile: + f = files.WebFile(reader, parent, name=name, is_dependency=is_dependency) + else: + f = reader + return f diff --git a/UnityPy/helpers/ResourceReader.py b/UnityPy/helpers/ResourceReader.py index be0cae4a..b624957a 100644 --- a/UnityPy/helpers/ResourceReader.py +++ b/UnityPy/helpers/ResourceReader.py @@ -1,4 +1,4 @@ -import os, glob +import ntpath from ..streams import EndianBinaryReader from ..files import File @@ -19,59 +19,33 @@ def get_resource_data(*args): -> -2 = offset, -1 = size """ if len(args) == 4: - reader = search_resource(res_path=args[0], assets_file=args[1]) + res_path, assets_file, offset, size = args + basename = ntpath.basename(res_path) + name, ext = ntpath.splitext(basename) + possible_names = { + basename, + f"{name}.resource", + f"{name}.assets.resS", + f"{name}.resS", + } + environment = assets_file.environment + reader = None + for possible_name in possible_names: + reader = environment.get_cab(possible_name) + if reader: + break + if not reader: + assets_file.load_dependencies(possible_names) + for possible_name in possible_names: + reader = environment.get_cab(possible_name) + if reader: + break + if not reader: + raise FileNotFoundError(f"Resource file {basename} not found") elif len(args) == 3: - reader = args[0] + reader, offset, size = args else: raise TypeError(f"3 or 4 arguments required, but only {len(args)} given") - reader.Position = args[-2] - return reader.read_bytes(args[-1]) - - -def search_resource(res_path, assets_file): - # try to find the resource in the Unity packages - base_name = os.path.basename(res_path) - if os.path.splitext(base_name)[1] == ".resource": - base_name2 = base_name.replace('.resource', '.assets.resS') - else: - base_name2 = base_name.replace('.assets.resS', '.resource') - - for p in [res_path, base_name, base_name2]: - reader = assets_file.parent.files.get(p) - if reader: - if isinstance(reader, File.File): - # in case the import helper accidentally detected a resource file as something else - reader = reader.reader - return reader - - # try to find it in the dir environment - c = assets_file - path = getattr(assets_file, "path", None) - while not path: - c = getattr(c,"parent",None) - if c == None: - raise FileNotFoundError( - f"Can't find the resource file {res_path}" - ) - path = getattr(c, "path", None) - current_directory = path - resource_file_path = os.path.join(current_directory, *res_path.split("/")) - if not os.path.isfile(resource_file_path): - resource_file_path = search_resource_file(current_directory, base_name) - if not os.path.isfile(resource_file_path): - resource_file_path = search_resource_file(current_directory, base_name.replace('.assets.resS', '.resource')) - - if os.path.isfile(resource_file_path): - return EndianBinaryReader(open(resource_file_path, "rb")) - else: - raise FileNotFoundError( - f"Can't find the resource file {res_path}" - ) - - -def search_resource_file(path, name): - #print("real file", path, name) - files = glob.glob(os.path.join(path, "**", name), recursive=True) - return files[0] if len(files) else "" - + reader.Position = offset + return reader.read_bytes(size) From 3b3d74d53da0ea7493ef2a199a1e81e07369a549 Mon Sep 17 00:00:00 2001 From: K0lb3 Date: Sat, 28 Jan 2023 22:17:43 +0100 Subject: [PATCH 05/14] SerializedFile - rework container handling --- UnityPy/files/SerializedFile.py | 91 +++++++++++++++++++++++++++------ 1 file changed, 75 insertions(+), 16 deletions(-) diff --git a/UnityPy/files/SerializedFile.py b/UnityPy/files/SerializedFile.py index 1248bfb3..37d10e38 100644 --- a/UnityPy/files/SerializedFile.py +++ b/UnityPy/files/SerializedFile.py @@ -179,10 +179,10 @@ class SerializedFile(File.File): types: list script_types: list externals: list - _container: dict objects: dict - container_: dict _cache: dict + assetbundle: "AssetBundle" + container: "ContainerHelper" header: SerializedFileHeader @property @@ -195,8 +195,8 @@ def files(self): def files(self, value): self.objects = value - def __init__(self, reader: EndianBinaryReader, parent=None, name=None): - super().__init__(parent=parent, name=name) + def __init__(self, reader: EndianBinaryReader, parent=None, name=None, **kwargs): + super().__init__(parent=parent, name=name, **kwargs) self.reader = reader self.unity_version = "2.5.0f5" @@ -207,10 +207,7 @@ def __init__(self, reader: EndianBinaryReader, parent=None, name=None): self.types = [] self.script_types = [] self.externals = [] - self._container = {} - self.objects = {} - self.container_ = {} # used to speed up mass asset extraction # some assets refer to each other, so by keeping the result # of specific assets cached the extraction can be speed up by a lot. @@ -291,18 +288,33 @@ def __init__(self, reader: EndianBinaryReader, parent=None, name=None): # read the asset_bundles to get the containers for obj in self.objects.values(): if obj.type == ClassIDType.AssetBundle: - data = obj.read() - for container, asset_info in data.m_Container.items(): - asset = asset_info.asset - self.container_[container] = asset - if hasattr(asset, "path_id"): - self._container[asset.path_id] = container - # if environment is not None: - # environment.container = {**environment.container, **self.container} + self.assetbundle = obj.read_typetree(wrap=True) + self._container = ContainerHelper(self.assetbundle.m_Container) + break + else: + self.assetbundle = None + self._container = ContainerHelper({}) @property def container(self): - return self.container_ + return self._container + + def load_dependencies(self, possible_dependencies: list = []): + """Load all external dependencies. + + Parameters + ---------- + possible_dependencies : list + List of possible dependencies for cases + where the target file is not listed as external. + """ + for file_id in self.externals: + self.environment.load_file(file_id.path, True) + for dependency in possible_dependencies: + try: + self.environment.load_file(dependency, True) + except FileNotFoundError: + pass def set_version(self, string_version): self.unity_version = string_version @@ -630,3 +642,50 @@ def read_string(string_buffer_reader: EndianBinaryReader, value: int) -> str: offset = value & 0x7FFFFFFF return CommonString.get(offset, str(offset)) + + +class ContainerHelper: + """Helper class to allow multidict containers + without breaking compatibility with old versions""" + + def __init__(self, container) -> None: + self.container = container + # support for getitem + self.container_dict = {key: value.asset for key, value in container} + self.path_dict = {value.asset.path_id: value.asset for key, value in container} + + def items(self): + return ((key, value.asset) for key, value in self.container) + + def keys(self): + return list({key for key, value in self.container}) + + def values(self): + return list({value.asset for key, value in self.container}) + + def __getitem__(self, key): + return self.container_dict[key] + + def __setitem__(self, key, value): + raise NotImplementedError("Assigning to container is not allowed!") + + def __delitem__(self, key): + raise NotImplementedError("Deleting from the container is not allowed!") + + def __iter__(self): + return iter(self.keys) + + def __len__(self): + return len(self.container) + + def __getattr__(self, name: str): + return self.container_dict[name] + + def __or__(self, other: "ContainerHelper"): + return ContainerHelper(list(set(self.container + other.container))) + + def __str__(self): + return f'{{{", ".join(f"{key}: {value}" for key, value in self.items())}}}' + + def __dict__(self): + return self.container_dict From 290f07be137777f4c103de9c1bd8797fdfcb6394 Mon Sep 17 00:00:00 2001 From: K0lb3 Date: Sat, 28 Jan 2023 22:18:10 +0100 Subject: [PATCH 06/14] Object - use new container handling and fix NodeHelper error with tuples --- UnityPy/classes/Object.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/UnityPy/classes/Object.py b/UnityPy/classes/Object.py index 2bfd844b..128976e1 100644 --- a/UnityPy/classes/Object.py +++ b/UnityPy/classes/Object.py @@ -5,7 +5,6 @@ from ..files import ObjectReader import types from ..exceptions import TypeTreeError as TypeTreeError -from .. import classes class Object(object): @@ -26,11 +25,7 @@ def __init__(self, reader: ObjectReader): if self.platform == BuildTarget.NoTarget: self._object_hide_flags = reader.read_u_int() - self.container = ( - self.assets_file._container[self.path_id] - if self.path_id in self.assets_file._container - else None - ) + self.container = self.assets_file.container.path_dict.get(self.path_id) self.reader.reset() if type(self) == Object: @@ -145,6 +140,8 @@ def __new__(cls, data, assets_file): return super(NodeHelper, cls).__new__(cls) elif isinstance(data, list): return [NodeHelper(x, assets_file) for x in data] + elif isinstance(data, tuple): + return tuple(NodeHelper(x, assets_file) for x in data) return data def __getitem__(self, item): From 17e7347907f7cd4dd3118d9146cbbe49ac1cef5b Mon Sep 17 00:00:00 2001 From: K0lb3 Date: Sat, 28 Jan 2023 22:19:39 +0100 Subject: [PATCH 07/14] add optional wrap argument for reading typetrees --- UnityPy/classes/Object.py | 4 ++-- UnityPy/files/ObjectReader.py | 13 ++++++------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/UnityPy/classes/Object.py b/UnityPy/classes/Object.py index 128976e1..2c8381ee 100644 --- a/UnityPy/classes/Object.py +++ b/UnityPy/classes/Object.py @@ -41,10 +41,10 @@ def dump_typetree(self, nodes: list = None) -> str: def dump_typetree_structure(self) -> str: return self.reader.dump_typetree_structure() - def read_typetree(self, nodes: list = None) -> dict: + def read_typetree(self, nodes: list = None, wrap: bool = False) -> dict: tree = self.reader.read_typetree(nodes) self.type_tree = NodeHelper(tree, self.assets_file) - return tree + return self.type_tree if wrap else tree def save_typetree(self, nodes: list = None, writer: EndianBinaryWriter = None): def class_to_dict(value): diff --git a/UnityPy/files/ObjectReader.py b/UnityPy/files/ObjectReader.py index 622e310d..2ebee31a 100644 --- a/UnityPy/files/ObjectReader.py +++ b/UnityPy/files/ObjectReader.py @@ -156,7 +156,7 @@ def Position(self, pos): def reset(self): self.reader.Position = self.byte_start - def read(self, return_typetree_on_error: bool=True): + def read(self, return_typetree_on_error: bool = True): cls = getattr(classes, self.type.name, None) obj = None @@ -171,9 +171,7 @@ def read(self, return_typetree_on_error: bool=True): else: raise e if not obj: - typetree = self.read_typetree() - if typetree: - obj = NodeHelper(typetree, self.assets_file) + obj = self.read_typetree(wrap=True) self._read_until = self.reader.Position return obj @@ -206,7 +204,7 @@ def dump_typetree_structure(self) -> str: def get_typetree_nodes(self, nodes: list = None) -> list: if nodes: return nodes - + if self.serialized_type: nodes = self.serialized_type.nodes if not nodes: @@ -215,10 +213,11 @@ def get_typetree_nodes(self, nodes: list = None) -> list: raise TypeTreeError("There are no TypeTree nodes for this object.") return nodes - def read_typetree(self, nodes: list = None) -> dict: + def read_typetree(self, nodes: list = None, wrap: bool = False) -> dict: self.reset() nodes = self.get_typetree_nodes(nodes) - return TypeTreeHelper.read_typetree(nodes, self) + res = TypeTreeHelper.read_typetree(nodes, self) + return NodeHelper(res, self.assets_file) if wrap else res def save_typetree( self, tree: dict, nodes: list = None, writer: EndianBinaryWriter = None From e764a7db4af1858474b07274a84e5ba15e4f51dc Mon Sep 17 00:00:00 2001 From: K0lb3 Date: Sat, 28 Jan 2023 22:28:28 +0100 Subject: [PATCH 08/14] File - fix read_files --- UnityPy/files/File.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/UnityPy/files/File.py b/UnityPy/files/File.py index d42d4460..2d496b9d 100644 --- a/UnityPy/files/File.py +++ b/UnityPy/files/File.py @@ -72,11 +72,11 @@ def read_files(self, reader: EndianBinaryReader, files: list): for node in files: reader.Position = node.offset name = node.path - reader = EndianBinaryReader( + node_reader = EndianBinaryReader( reader.read(node.size), offset=(reader.BaseOffset + node.offset) ) f = ImportHelper.parse_file( - reader, self.parent, name, is_dependency=self.is_dependency + node_reader, self.parent, name, is_dependency=self.is_dependency ) if isinstance(f, (EndianBinaryReader, SerializedFile.SerializedFile)): From 931fda673206cb39d3bb71b7fbaf2510401a2720 Mon Sep 17 00:00:00 2001 From: K0lb3 Date: Sun, 29 Jan 2023 14:27:27 +0100 Subject: [PATCH 09/14] implement custom filesystem handling --- README.md | 19 +++++++++- UnityPy/__init__.py | 6 ++-- UnityPy/environment.py | 50 ++++++++++++++------------ UnityPy/export/MeshRendererExporter.py | 25 ++++++++----- UnityPy/files/SerializedFile.py | 4 +-- pyproject.toml | 2 ++ 6 files changed, 68 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index f29401d7..aa86d88f 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,8 @@ if UnityPy.__version__ != '1.9.6': 2. [Example](#example) 3. [Important Classes](#important-classes) 4. [Important Object Types](#important-object-types) -5. [Credits](#credits) +5. [Custom Fileystem](#custom-filesystem) +6. [Credits](#credits) ## Installation @@ -364,6 +365,22 @@ if mesh_renderer.m_GameObject: mesh_renderer.export(export_dir) ``` +## Custom-Filesystem + +UnityPy uses [fsspec](https://github.com/fsspec/filesystem_spec) under the hood to manage all filesystem interactions. +This allows using various different types of filesystems without having to change UnityPy's code. +It also means that you can use your own custom filesystem to e.g. handle indirection via catalog files, load assets on demand from a server, or decrypt files. + +Following methods of the filesystem have to be implemented for using it in UnityPy. + +- sep (not a function, just the seperator as character) +- isfile(self, path) +- isdir(self, path) +- walk(self, path, \*\*kwargs) +- exists(self, path, \*\*kwargs) +- open(self, path, mode, \*\*kwargs) ("rb" mode required, "wt" required for ModelExporter) +- makedirs(self, path, exist_ok=False) + ## Credits First of all, diff --git a/UnityPy/__init__.py b/UnityPy/__init__.py index c7f92f4c..a46b869a 100644 --- a/UnityPy/__init__.py +++ b/UnityPy/__init__.py @@ -1,11 +1,11 @@ -__version__ = "1.9.24" +__version__ = "1.10.00" from .environment import Environment from .helpers.ArchiveStorageManager import set_assetbundle_decrypt_key -def load(*args): - return Environment(*args) +def load(*args, fs=None, **kwargs): + return Environment(*args, fs=fs, **kwargs) # backward compatibility diff --git a/UnityPy/environment.py b/UnityPy/environment.py index 093e0eed..7a60130e 100644 --- a/UnityPy/environment.py +++ b/UnityPy/environment.py @@ -1,15 +1,18 @@ -from typing import List, Callable, Dict, Union import io import os import ntpath -from zipfile import ZipFile import re -from . import files -from .files import File, ObjectReader +from typing import List, Callable, Dict, Union +from zipfile import ZipFile + +from fsspec import AbstractFileSystem +from fsspec.implementations.local import LocalFileSystem + + +from .files import File, ObjectReader, SerializedFile from .enums import FileType from .helpers import ImportHelper from .streams import EndianBinaryReader -from .files import SerializedFile reSplit = re.compile(r"(.*?([^\/\\]+?))\.split\d+") @@ -21,27 +24,27 @@ class Environment: local_files: List[str] local_files_simple: List[str] - def __init__(self, *args): + def __init__(self, *args, fs: AbstractFileSystem = None): self.files = {} self.cabs = {} self.path = None - self.out_path = os.path.join(os.getcwd(), "output") + self.fs = fs or LocalFileSystem() self.local_files = [] self.local_files_simple = [] if args: for arg in args: if isinstance(arg, str): - if os.path.isfile(arg): - if os.path.splitext(arg)[-1] in [".apk", ".zip"]: + if self.fs.isfile(arg): + if ntpath.splitext(arg)[-1] in [".apk", ".zip"]: self.load_zip_file(arg) else: - self.path = os.path.dirname(arg) + self.path = ntpath.dirname(arg) if reSplit.match(arg): self.load_files([arg]) else: self.load_file(arg) - elif os.path.isdir(arg): + elif self.fs.isdir(arg): self.path = arg self.load_folder(arg) else: @@ -62,8 +65,8 @@ def load_folder(self, path: str): """Loads all files in the given path and its subdirs into the Environment.""" self.load_files( [ - os.path.join(root, f) - for root, dirs, files in os.walk(path) + self.fs.sep.join([root, f]) + for root, dirs, files in self.fs.walk(path) for f in files ] ) @@ -72,9 +75,9 @@ def load(self, files: list): """Loads all files into the Environment.""" self.files.update( { - os.path.basename(f): self.load_file(open(f, "rb"), self, f) + ntpath.basename(f): self.load_file(self.fs.open(f, "rb"), self, f) for f in files - if os.path.exists(f) + if self.fs.exists(f) } ) @@ -87,6 +90,7 @@ def load_file( ): if not parent: parent = self + if isinstance(file, str): split_match = reSplit.match(file) if split_match: @@ -94,8 +98,8 @@ def load_file( file = [] for i in range(0, 999): item = f"{basepath}.split{i}" - if item in files: - with open(item, "rb") as f: + if self.fs.exists(item): + with self.fs.open(item, "rb") as f: file.append(f.read()) elif i: break @@ -103,7 +107,7 @@ def load_file( file = b"".join(file) else: name = file - file = open(file, "rb") + file = self.fs.open(file, "rb") typ, reader = ImportHelper.check_file_type(file) @@ -132,7 +136,7 @@ def load_file( def load_zip_file(self, value): buffer = None - if isinstance(value, str) and os.path.exists(value): + if isinstance(value, str) and self.fs.exists(value): buffer = open(value, "rb") elif isinstance(value, (bytes, bytearray)): buffer = io.BytesIO(value) @@ -143,7 +147,7 @@ def load_zip_file(self, value): self.load_assets(z.namelist(), lambda x: z.open(x, "r")) z.close() - def save(self, pack="none"): + def save(self, pack="none", out_path="output"): """Saves all changed assets. Mark assets as changed using `.mark_changed()`. pack = "none" (default) or "lz4" @@ -151,7 +155,7 @@ def save(self, pack="none"): for f in self.files: if self.files[f].is_changed: with open( - os.path.join(self.out_path, os.path.basename(f)), "wb" + self.fs.sep.join([out_path, ntpath.basename(f)]), "wb" ) as out: out.write(self.files[f].save(packer=pack)) @@ -295,9 +299,9 @@ def find_file(self, name: str, is_dependency: bool = True) -> Union[File, None]: return cab if len(self.local_files) == 0 and self.path: - for root, _, files in os.walk(self.path): + for root, _, files in self.fs.walk(self.path): for name in files: - self.local_files.append(os.path.join(root, name)) + self.local_files.append(self.fs.sep.join([root, name])) if name in self.local_files: fp = name diff --git a/UnityPy/export/MeshRendererExporter.py b/UnityPy/export/MeshRendererExporter.py index 89992b22..233b5878 100644 --- a/UnityPy/export/MeshRendererExporter.py +++ b/UnityPy/export/MeshRendererExporter.py @@ -1,6 +1,5 @@ import os -from ..classes import Renderer, SkinnedMeshRenderer, Material, Texture2D -from ..enums import ClassIDType +from ..classes import Renderer, SkinnedMeshRenderer, Material from .MeshExporter import export_mesh_obj @@ -18,7 +17,8 @@ def get_mesh(meshR: Renderer): def export_mesh_renderer(obj: Renderer, export_dir: str) -> None: - os.makedirs(export_dir, exist_ok=True) + env = mesh.assets_file.enviroment + env.fs.makedirs(export_dir, exist_ok=True) meshR = obj.read() mesh = get_mesh(meshR) if not mesh: @@ -49,18 +49,25 @@ def export_mesh_renderer(obj: Renderer, export_dir: str) -> None: if not texEnv.m_Texture: continue tex = texEnv.m_Texture.read() - texName = f"{tex.name if tex.name else key}.png" - tex.read().image.save(os.path.join(export_dir, texName)) + texName = f"{tex.m_Name if tex.m_Name else key}.png" + with env.fs.open(env.fs.sep.join([export_dir, texName]), "wb") as f: + tex.read().image.save(f) # save .obj - with open( - os.path.join(export_dir, f"{mesh.name}.obj"), "wt", encoding="utf8", newline="" + with env.fs.open( + env.fs.sep.join([export_dir, f"{mesh.m_Name}.obj"]), + "wt", + encoding="utf8", + newline="", ) as f: f.write(export_mesh_obj(mesh, material_names)) # save .mtl - with open( - os.path.join(export_dir, f"{mesh.name}.mtl"), "wt", encoding="utf8", newline="" + with env.fs.open( + env.fs.sep.join([export_dir, f"{mesh.m_Name}.mtl"]), + "wt", + encoding="utf8", + newline="", ) as f: f.write("\n".join(materials)) diff --git a/UnityPy/files/SerializedFile.py b/UnityPy/files/SerializedFile.py index 37d10e38..314781ab 100644 --- a/UnityPy/files/SerializedFile.py +++ b/UnityPy/files/SerializedFile.py @@ -1,4 +1,4 @@ -import os +from ntpath import basename import re from . import File, ObjectReader @@ -57,7 +57,7 @@ class FileIdentifier: # external @property def name(self): - return os.path.basename(self.path) + return basename(self.path) def __repr__(self): return f"<{self.__class__.__name__}({self.path})>" diff --git a/pyproject.toml b/pyproject.toml index aab8463c..3f219daa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,8 @@ dependencies = [ "tabulate", # audio extraction "pyfmodex", + # filesystem handling + "fsspec", ] dynamic = ["version"] From 75ac67e587f5f4170c53ef0f0c59b9062daee1ad Mon Sep 17 00:00:00 2001 From: K0lb3 Date: Sun, 29 Jan 2023 14:30:02 +0100 Subject: [PATCH 10/14] Update README.md --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index aa86d88f..4214b86b7 100644 --- a/README.md +++ b/README.md @@ -374,12 +374,12 @@ It also means that you can use your own custom filesystem to e.g. handle indirec Following methods of the filesystem have to be implemented for using it in UnityPy. - sep (not a function, just the seperator as character) -- isfile(self, path) -- isdir(self, path) -- walk(self, path, \*\*kwargs) -- exists(self, path, \*\*kwargs) -- open(self, path, mode, \*\*kwargs) ("rb" mode required, "wt" required for ModelExporter) -- makedirs(self, path, exist_ok=False) +- isfile(self, path: str) -> bool +- isdir(self, path: str) -> bool +- exists(self, path: str, \*\*kwargs) -> bool +- walk(self, path: str, \*\*kwargs) -> Iterable[List[str], List[str], List[str]] +- open(self, path: str, mode: str = "rb", \*\*kwargs) -> file ("rb" mode required, "wt" required for ModelExporter) +- makedirs(self, path: str, exist_ok: bool = False) -> bool ## Credits From 146944892cf012f48106aa0739da784766f755eb Mon Sep 17 00:00:00 2001 From: K0lb3 Date: Sun, 29 Jan 2023 17:15:20 +0100 Subject: [PATCH 11/14] EndianBinaryReader - fix passing of LocalFileOpenener --- UnityPy/streams/EndianBinaryReader.py | 28 +++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/UnityPy/streams/EndianBinaryReader.py b/UnityPy/streams/EndianBinaryReader.py index c89fdfbc..0fe7f7db 100644 --- a/UnityPy/streams/EndianBinaryReader.py +++ b/UnityPy/streams/EndianBinaryReader.py @@ -1,9 +1,8 @@ -import io import sys from struct import Struct, unpack import re from typing import List, Union -from io import BytesIO, BufferedIOBase +from io import BytesIO, BufferedIOBase, IOBase, BufferedReader reNot0 = re.compile(b"(.*?)\x00") @@ -50,14 +49,29 @@ def __new__( ): if isinstance(item, (bytes, bytearray, memoryview)): obj = super(EndianBinaryReader, cls).__new__(EndianBinaryReader_Memoryview) - elif isinstance(item, BufferedIOBase): + elif isinstance(item, (IOBase, BufferedIOBase)): obj = super(EndianBinaryReader, cls).__new__(EndianBinaryReader_Streamable) elif isinstance(item, str): item = open(item, "rb") obj = super(EndianBinaryReader, cls).__new__(EndianBinaryReader_Streamable) elif isinstance(item, EndianBinaryReader): - item = item.stream if isinstance(item, EndianBinaryReader_Streamable) else item.view + item = ( + item.stream + if isinstance(item, EndianBinaryReader_Streamable) + else item.view + ) return EndianBinaryReader(item, endian, offset) + elif hasattr(item, "read"): + if hasattr(item, "seek") and hasattr(item, "tell"): + obj = super(EndianBinaryReader, cls).__new__( + EndianBinaryReader_Streamable + ) + else: + item = item.read() + obj = super(EndianBinaryReader, cls).__new__( + EndianBinaryReader_Memoryview + ) + obj.__init__(item, endian) return obj @@ -287,7 +301,9 @@ def read_string_to_null(self, max_length=32767) -> str: if self.Position + max_length >= self.Length: raise Exception("String not terminated") else: - return bytes(self.read_bytes(max_length)).decode("utf8", "surrogateescape") + return bytes(self.read_bytes(max_length)).decode( + "utf8", "surrogateescape" + ) ret = match[1].decode("utf8", "surrogateescape") self.Position = match.end() return ret @@ -419,7 +435,7 @@ def read_vector4(self): class EndianBinaryReader_Streamable(EndianBinaryReader): __slots__ = ("stream", "_endian", "BaseOffset") - stream: io.BufferedReader + stream: BufferedReader def __init__(self, stream, endian=">", offset=0): self._endian = "" From 232937cd3240867bfefef3f5d04b14301fcb5ae3 Mon Sep 17 00:00:00 2001 From: Rudolf Kolbe Date: Mon, 5 Jun 2023 12:52:49 +0200 Subject: [PATCH 12/14] Readme.md - min python 3.6.0 -> 3.7.0 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4214b86b7..18149b6b 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ if UnityPy.__version__ != '1.9.6': ## Installation -**Python 3.6.0 or higher is required** +**Python 3.7.0 or higher is required** via pypi From 56b4426ffbb53940dd3f2d177dac7881154c431d Mon Sep 17 00:00:00 2001 From: Rudolf Kolbe Date: Mon, 5 Jun 2023 12:55:25 +0200 Subject: [PATCH 13/14] setup.py - possible legacy egg fix --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index b4adc9d3..23d54c66 100644 --- a/setup.py +++ b/setup.py @@ -56,6 +56,7 @@ def get_fmod_library(): setup( + name="UnityPy", packages=find_packages() + extra_packages, package_data={"UnityPy": unitypy_package_data}, ext_modules=[ From 135e3c0a1d5d131c5c1abef8d04de387b447303f Mon Sep 17 00:00:00 2001 From: Rudolf Kolbe Date: Mon, 5 Jun 2023 12:55:45 +0200 Subject: [PATCH 14/14] pyproject.toml - remove 3.6 support, add 3.10 and 3.11 support --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3f219daa..6e27f3a5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,10 +26,11 @@ classifiers = [ "Development Status :: 5 - Production/Stable", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Topic :: Software Development :: Libraries :: Python Modules", "Topic :: Games/Entertainment", "Topic :: Multimedia :: Graphics",