vpkpp: add xzp read-only support

include/vpkpp/format/ORE.h

@@ -1,7 +1,5 @@
 #pragma once
 
-#include <sourcepp/parser/Binary.h>
-
 #include "../PackFile.h"
 
 namespace vpkpp {

include/vpkpp/format/XZP.h

@@ -0,0 +1,35 @@
+#pragma once
+
+#include <sourcepp/parser/Binary.h>
+
+#include "../PackFile.h"
+
+namespace vpkpp {
+
+constexpr auto XZP_HEADER_SIGNATURE = sourcepp::parser::binary::makeFourCC("piZx");
+constexpr auto XZP_FOOTER_SIGNATURE = sourcepp::parser::binary::makeFourCC("tFzX");
+constexpr std::string_view XZP_EXTENSION = ".xzp";
+
+class XZP : public PackFileReadOnly {
+public:
+	/// Open an XZP file
+	[[nodiscard]] static std::unique_ptr<PackFile> open(const std::string& path, const EntryCallback& callback = nullptr);
+
+	static constexpr inline std::string_view GUID = "A682CF9BCA0A4980A920B5C00C8E0945";
+
+	[[nodiscard]] constexpr std::string_view getGUID() const override {
+		return XZP::GUID;
+	}
+
+	[[nodiscard]] std::optional<std::vector<std::byte>> readEntry(const std::string& path_) const override;
+
+	[[nodiscard]] Attribute getSupportedEntryAttributes() const override;
+
+protected:
+	using PackFileReadOnly::PackFileReadOnly;
+
+private:
+	VPKPP_REGISTER_PACKFILE_OPEN(XZP_EXTENSION, &XZP::open);
+};
+
+} // namespace vpkpp

include/vpkpp/vpkpp.h

@@ -18,6 +18,7 @@
 #include "format/VPK_VTMB.h"
 #include "format/VPP.h"
 #include "format/WAD3.h"
+#include "format/XZP.h"
 #include "format/ZIP.h"
 #include "Attribute.h"
 #include "Entry.h"

lang/c/include/vpkppc/format/XZP.h

@@ -0,0 +1,9 @@
+#pragma once
+
+#include "../PackFile.h"
+
+// REQUIRES MANUAL FREE: vpkpp_close
+SOURCEPP_API vpkpp_pack_file_handle_t vpkpp_xzp_open(const char* path, vpkpp_entry_callback_t callback);
+
+// REQUIRES MANUAL FREE: sourcepp_string_free
+SOURCEPP_API sourcepp_string_t vpkpp_xzp_guid(vpkpp_pack_file_handle_t handle);

lang/c/include/vpkppc/vpkpp.h

@@ -17,6 +17,7 @@
 #include "format/VPK.h"
 #include "format/VPK_VTMB.h"
 #include "format/WAD3.h"
+#include "format/XZP.h"
 #include "format/ZIP.h"
 #include "Attribute.h"
 #include "Entry.h"

lang/c/src/vpkppc/_vpkppc.cmake

@@ -13,6 +13,7 @@ add_pretty_parser(vpkpp C
         "${CMAKE_CURRENT_SOURCE_DIR}/lang/c/include/vpkppc/format/VPK_VTMB.h"
         "${CMAKE_CURRENT_SOURCE_DIR}/lang/c/include/vpkppc/format/VPP.h"
         "${CMAKE_CURRENT_SOURCE_DIR}/lang/c/include/vpkppc/format/WAD3.h"
+        "${CMAKE_CURRENT_SOURCE_DIR}/lang/c/include/vpkppc/format/XZP.h"
         "${CMAKE_CURRENT_SOURCE_DIR}/lang/c/include/vpkppc/format/ZIP.h"
         "${CMAKE_CURRENT_SOURCE_DIR}/lang/c/include/vpkppc/Attribute.h"
         "${CMAKE_CURRENT_SOURCE_DIR}/lang/c/include/vpkppc/Entry.h"
@@ -34,6 +35,7 @@ add_pretty_parser(vpkpp C
         "${CMAKE_CURRENT_LIST_DIR}/format/VPK_VTMB.cpp"
         "${CMAKE_CURRENT_LIST_DIR}/format/VPP.cpp"
         "${CMAKE_CURRENT_LIST_DIR}/format/WAD3.cpp"
+        "${CMAKE_CURRENT_LIST_DIR}/format/XZP.cpp"
         "${CMAKE_CURRENT_LIST_DIR}/format/ZIP.cpp"
         "${CMAKE_CURRENT_LIST_DIR}/Convert.cpp"
         "${CMAKE_CURRENT_LIST_DIR}/Entry.cpp"

lang/c/src/vpkppc/format/ORE.cpp

@@ -1,4 +1,4 @@
-#include <vpkppc/format/PAK.h>
+#include <vpkppc/format/ORE.h>
 
 #include <vpkpp/format/ORE.h>
 

lang/c/src/vpkppc/format/XZP.cpp

@@ -0,0 +1,27 @@
+#include <vpkppc/format/XZP.h>
+
+#include <vpkpp/format/XZP.h>
+
+#include <sourceppc/Convert.hpp>
+#include <sourceppc/Helpers.h>
+
+using namespace vpkpp;
+
+SOURCEPP_API vpkpp_pack_file_handle_t vpkpp_xzp_open(const char* path, vpkpp_entry_callback_t callback) {
+	SOURCEPP_EARLY_RETURN_VAL(path, nullptr);
+
+	auto packFile = XZP::open(path, callback ? [callback](const std::string& path, const Entry& entry) {
+		callback(path.c_str(), const_cast<Entry*>(&entry));
+	} : static_cast<PackFile::EntryCallback>(nullptr));
+	if (!packFile) {
+		return nullptr;
+	}
+	return packFile.release();
+}
+
+// REQUIRES MANUAL FREE: sourcepp_string_free
+SOURCEPP_API sourcepp_string_t vpkpp_xzp_guid(vpkpp_pack_file_handle_t handle) {
+	SOURCEPP_EARLY_RETURN_VAL(handle, SOURCEPP_STRING_INVALID);
+
+	return Convert::toString(XZP::GUID);
+}

lang/csharp/src/vpkpp/Format/XZP.cs

@@ -0,0 +1,58 @@
+using System;
+using System.Runtime.InteropServices;
+
+namespace vpkpp.Format
+{
+    using EntryCallback = Action<string, Entry>;
+
+    internal static unsafe partial class Extern
+    {
+		internal static unsafe partial class XZP
+		{
+			[LibraryImport("sourcepp_vpkppc", EntryPoint = "vpkpp_xzp_open")]
+			public static partial void* Open([MarshalAs(UnmanagedType.LPStr)] string path, IntPtr callback);
+
+			[LibraryImport("sourcepp_vpkppc", EntryPoint = "vpkpp_xzp_guid")]
+			public static partial sourcepp.String GUID();
+		}
+    }
+
+    public class XZP : PackFile
+    {
+        private protected unsafe XZP(void* handle) : base(handle) {}
+
+        public new static XZP? Open(string path)
+        {
+            unsafe
+            {
+                var handle = Extern.XZP.Open(path, 0);
+                return handle == null ? null : new XZP(handle);
+            }
+        }
+
+        public new static XZP? Open(string path, EntryCallback callback)
+        {
+            unsafe
+            {
+                EntryCallbackNative callbackNative = (path, entry) =>
+                {
+                    callback(path, new Entry(entry, true));
+                };
+                var handle = Extern.XZP.Open(path, Marshal.GetFunctionPointerForDelegate(callbackNative));
+                return handle == null ? null : new XZP(handle);
+            }
+        }
+
+		public static string GUID
+		{
+			get
+			{
+				unsafe
+				{
+					var str = Extern.XZP.GUID();
+					return sourcepp.StringUtils.ConvertToStringAndDelete(ref str);
+				}
+			}
+		}
+	}
+}

src/vpkpp/_vpkpp.cmake

@@ -15,6 +15,7 @@ add_pretty_parser(vpkpp
         "${CMAKE_CURRENT_SOURCE_DIR}/include/vpkpp/format/VPK_VTMB.h"
         "${CMAKE_CURRENT_SOURCE_DIR}/include/vpkpp/format/VPP.h"
         "${CMAKE_CURRENT_SOURCE_DIR}/include/vpkpp/format/WAD3.h"
+        "${CMAKE_CURRENT_SOURCE_DIR}/include/vpkpp/format/XZP.h"
         "${CMAKE_CURRENT_SOURCE_DIR}/include/vpkpp/format/ZIP.h"
         "${CMAKE_CURRENT_SOURCE_DIR}/include/vpkpp/Attribute.h"
         "${CMAKE_CURRENT_SOURCE_DIR}/include/vpkpp/Entry.h"
@@ -35,6 +36,7 @@ add_pretty_parser(vpkpp
         "${CMAKE_CURRENT_LIST_DIR}/format/VPK_VTMB.cpp"
         "${CMAKE_CURRENT_LIST_DIR}/format/VPP.cpp"
         "${CMAKE_CURRENT_LIST_DIR}/format/WAD3.cpp"
+        "${CMAKE_CURRENT_LIST_DIR}/format/XZP.cpp"
         "${CMAKE_CURRENT_LIST_DIR}/format/ZIP.cpp"
         "${CMAKE_CURRENT_LIST_DIR}/PackFile.cpp")
 

src/vpkpp/format/XZP.cpp

@@ -0,0 +1,158 @@
+#include <vpkpp/format/XZP.h>
+
+#include <filesystem>
+
+#include <FileStream.h>
+
+using namespace sourcepp;
+using namespace vpkpp;
+
+std::unique_ptr<PackFile> XZP::open(const std::string& path, const EntryCallback& callback) {
+	if (!std::filesystem::exists(path)) {
+		// File does not exist
+		return nullptr;
+	}
+
+	auto* xzp = new XZP{path};
+	auto packFile = std::unique_ptr<PackFile>(xzp);
+
+	FileStream reader{xzp->fullFilePath};
+	reader.seek_in(0);
+
+	if (reader.read<uint32_t>() != XZP_HEADER_SIGNATURE) {
+		// File is not an XZP
+		return nullptr;
+	}
+
+	if (reader.read<uint32_t>() != 6) {
+		// Invalid version - check around for earlier formats eventually
+		return nullptr;
+	}
+
+	const auto preloadDirectoryEntryCount = reader.read<uint32_t>();
+	const auto directoryEntryCount = reader.read<uint32_t>();
+	reader.skip_in<uint32_t>(); // preloadBytes
+
+	if (reader.read<uint32_t>() != sizeof(uint32_t) * 9) {
+		// Header size - should always be 9 uints for v6
+		return nullptr;
+	}
+
+	if (const auto filepathEntryCount = reader.read<uint32_t>(); filepathEntryCount != directoryEntryCount) {
+		// We can't reverse a hash! Just bail
+		return nullptr;
+	}
+
+	const auto filepathStringsOffset = reader.read<uint32_t>();
+	reader.skip_in<uint32_t>(); // filepathStringsLength
+
+	// Add directory entries
+	std::unordered_map<uint32_t, std::vector<std::pair<uint32_t, uint32_t>>> stagedEntryChunks;
+	for (uint32_t i = 0; i < directoryEntryCount; i++) {
+		const auto filepathCRC = reader.read<uint32_t>();
+		const auto chunkLength = reader.read<uint32_t>();
+		const auto chunkOffset = reader.read<uint32_t>();
+
+		if (!stagedEntryChunks.contains(filepathCRC)) {
+			stagedEntryChunks[filepathCRC] = {};
+			stagedEntryChunks[filepathCRC].emplace_back(chunkOffset, chunkLength);
+		} else if (stagedEntryChunks[filepathCRC].back().first + stagedEntryChunks[filepathCRC].back().second == chunkOffset) {
+			stagedEntryChunks[filepathCRC].back().second += chunkLength;
+		} else {
+			stagedEntryChunks[filepathCRC].emplace_back(chunkOffset, chunkLength);
+		}
+	}
+
+	// Add preload entries
+	std::unordered_map<uint32_t, std::pair<uint32_t, uint32_t>> stagedEntryPreloads;
+	for (uint32_t i = 0; i < preloadDirectoryEntryCount; i++) {
+		const auto filepathCRC = reader.read<uint32_t>();
+		const auto preloadLength = reader.read<uint32_t>();
+		const auto preloadOffset = reader.read<uint32_t>();
+
+		stagedEntryPreloads[filepathCRC] = {preloadLength, preloadOffset};
+	}
+
+	// Preload size per entry
+	reader.skip_in<uint16_t>(directoryEntryCount);
+
+	// filepaths, and put it all together simultaneously
+	reader.seek_in(filepathStringsOffset);
+	std::unordered_map<uint32_t, std::string> stagedEntryFilepaths;
+	for (uint32_t i = 0; i < directoryEntryCount; i++) {
+		const auto filepathCRC = reader.read<uint32_t>();
+		const auto filepathOffset = reader.read<uint32_t>();
+		reader.skip_in<uint32_t>(); // timestamp
+
+		const auto readerPos = reader.tell_in();
+		reader.seek_in_u(filepathOffset);
+		stagedEntryFilepaths[filepathCRC] = reader.read_string();
+		reader.seek_in_u(readerPos);
+	}
+
+	// Put it all together
+	for (const auto& [filepathCRC, filepath] : stagedEntryFilepaths) {
+		Entry entry = createNewEntry();
+
+		auto entryPath = xzp->cleanEntryPath(filepath);
+
+		BufferStream stream{entry.extraData};
+
+		const auto& chunks = stagedEntryChunks[filepathCRC];
+		stream.write<uint32_t>(chunks.size());
+
+		entry.length = 0;
+		for (const auto& chunk : chunks) {
+			entry.length += chunk.second;
+			stream << chunk.first << chunk.second;
+		}
+
+		if (stagedEntryPreloads.contains(filepathCRC)) {
+			const auto& preload = stagedEntryPreloads[filepathCRC];
+			stream.write<uint32_t>(true);
+			stream << preload.first << preload.second;
+		} else {
+			stream.write<uint32_t>(false);
+		}
+
+		xzp->entries.emplace(entryPath, entry);
+
+		if (callback) {
+			callback(entryPath, entry);
+		}
+	}
+
+	return packFile;
+}
+
+std::optional<std::vector<std::byte>> XZP::readEntry(const std::string& path_) const {
+	auto path = this->cleanEntryPath(path_);
+	auto entry = this->findEntry(path);
+	if (!entry) {
+		return std::nullopt;
+	}
+	if (entry->unbaked) {
+		return readUnbakedEntry(*entry);
+	}
+
+	// It's baked into the file on disk
+	FileStream stream{this->fullFilePath};
+	if (!stream) {
+		return std::nullopt;
+	}
+	std::vector<std::byte> out;
+	BufferStreamReadOnly entryDataStream{entry->extraData};
+	const auto chunks = entryDataStream.read<uint32_t>();
+	for (uint32_t i = 0; i < chunks; i++) {
+		const auto chunkOffset = entryDataStream.read<uint32_t>();
+		const auto chunkLength = entryDataStream.read<uint32_t>();
+		const auto chunkData = stream.seek_in(chunkOffset).read_bytes(chunkLength);
+		out.insert(out.end(), chunkData.begin(), chunkData.end());
+	}
+	return out;
+}
+
+Attribute XZP::getSupportedEntryAttributes() const {
+	using enum Attribute;
+	return LENGTH;
+}