From 8265d83aa240671bada64db4339f558dd93690fc Mon Sep 17 00:00:00 2001 From: ozxybox Date: Sun, 29 Sep 2024 20:37:43 -0500 Subject: [PATCH 1/3] feat(vpkpp): add WAD3 support --- include/vpkpp/PackFileType.h | 1 + include/vpkpp/format/WAD3.h | 39 ++++++ include/vpkpp/vpkpp.h | 1 + src/vpkpp/_vpkpp.cmake | 2 + src/vpkpp/format/WAD3.cpp | 235 +++++++++++++++++++++++++++++++++++ 5 files changed, 278 insertions(+) create mode 100644 include/vpkpp/format/WAD3.h create mode 100644 src/vpkpp/format/WAD3.cpp diff --git a/include/vpkpp/PackFileType.h b/include/vpkpp/PackFileType.h index 2d9454926..9494bb5de 100644 --- a/include/vpkpp/PackFileType.h +++ b/include/vpkpp/PackFileType.h @@ -12,6 +12,7 @@ enum class PackFileType { PCK, VPK, VPK_VTMB, + WAD3, ZIP, }; diff --git a/include/vpkpp/format/WAD3.h b/include/vpkpp/format/WAD3.h new file mode 100644 index 000000000..41b08ed7c --- /dev/null +++ b/include/vpkpp/format/WAD3.h @@ -0,0 +1,39 @@ +#pragma once + +#include + +#include "../PackFile.h" + +namespace vpkpp { + +constexpr int8_t WAD3_FILENAME_MAX_SIZE = 16; +constexpr auto WAD3_SIGNATURE = sourcepp::parser::binary::makeFourCC("WAD3"); +constexpr std::string_view WAD3_EXTENSION = ".wad"; + +/** + * Valve GoldSrc WAD3 file + */ +class WAD3 : public PackFile { +public: + /// Create a WAD3 file + static std::unique_ptr create(const std::string& path); + + /// Open a WAD3 file + [[nodiscard]] static std::unique_ptr open(const std::string& path, const EntryCallback& callback = nullptr); + + [[nodiscard]] std::optional> readEntry(const std::string& path_) const override; + + bool bake(const std::string& outputDir_ /*= ""*/, BakeOptions options /*= {}*/, const EntryCallback& callback /*= nullptr*/) override; + + [[nodiscard]] Attribute getSupportedEntryAttributes() const override; + +protected: + explicit WAD3(const std::string& fullFilePath_); + + void addEntryInternal(Entry& entry, const std::string& path, std::vector& buffer, EntryOptions options) override; + +private: + VPKPP_REGISTER_PACKFILE_OPEN(WAD3_EXTENSION, &WAD3::open); +}; + +} // namespace vpkpp diff --git a/include/vpkpp/vpkpp.h b/include/vpkpp/vpkpp.h index 4b29437e0..71214acc5 100644 --- a/include/vpkpp/vpkpp.h +++ b/include/vpkpp/vpkpp.h @@ -13,6 +13,7 @@ #include "format/PCK.h" #include "format/VPK.h" #include "format/VPK_VTMB.h" +#include "format/WAD3.h" #include "format/ZIP.h" #include "Attribute.h" #include "Entry.h" diff --git a/src/vpkpp/_vpkpp.cmake b/src/vpkpp/_vpkpp.cmake index 91391aaef..b1746cb12 100644 --- a/src/vpkpp/_vpkpp.cmake +++ b/src/vpkpp/_vpkpp.cmake @@ -10,6 +10,7 @@ add_pretty_parser(vpkpp "${CMAKE_CURRENT_SOURCE_DIR}/include/vpkpp/format/PCK.h" "${CMAKE_CURRENT_SOURCE_DIR}/include/vpkpp/format/VPK.h" "${CMAKE_CURRENT_SOURCE_DIR}/include/vpkpp/format/VPK_VTMB.h" + "${CMAKE_CURRENT_SOURCE_DIR}/include/vpkpp/format/WAD3.h" "${CMAKE_CURRENT_SOURCE_DIR}/include/vpkpp/format/ZIP.h" "${CMAKE_CURRENT_SOURCE_DIR}/include/vpkpp/Attribute.h" "${CMAKE_CURRENT_SOURCE_DIR}/include/vpkpp/Entry.h" @@ -26,6 +27,7 @@ add_pretty_parser(vpkpp "${CMAKE_CURRENT_LIST_DIR}/format/PCK.cpp" "${CMAKE_CURRENT_LIST_DIR}/format/VPK.cpp" "${CMAKE_CURRENT_LIST_DIR}/format/VPK_VTMB.cpp" + "${CMAKE_CURRENT_LIST_DIR}/format/WAD3.cpp" "${CMAKE_CURRENT_LIST_DIR}/format/ZIP.cpp" "${CMAKE_CURRENT_LIST_DIR}/PackFile.cpp") diff --git a/src/vpkpp/format/WAD3.cpp b/src/vpkpp/format/WAD3.cpp new file mode 100644 index 000000000..ac388bd15 --- /dev/null +++ b/src/vpkpp/format/WAD3.cpp @@ -0,0 +1,235 @@ +// +// Header: (size=12) +// fourcc identity +// int32 lump_count // Number of lump infos in the array +// int32 lump_offset // Offset to the start of the lump info array +// +// NOTE: Are the compression fields actually used in WAD3? Answer seems to be no +// Lump Infos: (size=32) +// int32 offset +// int32 size +// int32 size_uncompressed +// int8 type +// int8 compression +// int16 padding +// char name[16] + +#include + +#include + +#include + +#include +#include + +using namespace sourcepp; +using namespace vpkpp; + +enum WAD3FileTypes +{ + WFT_FIRST = 64, + + WFT_PALETTE = WFT_FIRST, + WFT_COLORMAP, + WFT_QPIC, + WFT_MIPTEX, + WFT_RAW, + WFT_COLORMAP2, + WFT_FONT, + + WFT_COUNT, +}; + +constexpr std::string_view k_FileTypeNames[] = { + "palette", + "colormap", + "qpic", + "miptex", + "raw", + "colormap2", + "font", +}; + +WAD3::WAD3(const std::string& fullFilePath_) + : PackFile(fullFilePath_) { + this->type = PackFileType::WAD3; +} + +std::unique_ptr WAD3::create(const std::string& path) { + FileStream stream{path, FileStream::OPT_TRUNCATE | FileStream::OPT_CREATE_IF_NONEXISTENT}; + stream + .write(WAD3_SIGNATURE) + .write(0) + .write(0); + return WAD3::open(path); +} + +std::unique_ptr WAD3::open(const std::string& path, const EntryCallback& callback) { + if (!std::filesystem::exists(path)) { + // File does not exist + return nullptr; + } + + auto* wad = new WAD3{path}; + std::unique_ptr packFile{wad}; + + FileStream reader{wad->fullFilePath}; + reader.seek_in(0); + + // Verify the identity + if (auto signature = reader.read(); signature != WAD3_SIGNATURE) { + return nullptr; + } + + // Treating counts as unsigned to simplify some logic... No reason to really have a negative count here anyways + auto lumpCount = reader.read(); + auto lumpOffset = reader.read(); + + // Read in all lump entries + reader.seek_in(lumpOffset); + for (uint32_t i = 0; i < lumpCount; i++) { + // Read all entry data + auto offset = reader.read(); + auto size = reader.read(); + reader.skip_in(); // size_uncompressed + auto type = reader.read(); + reader.skip_in(); // compression + reader.skip_in(); // padding + auto name = reader.read_string(WAD3_FILENAME_MAX_SIZE); + + // We'll append the type onto the name as an extension + if(type >= WFT_FIRST && type < WFT_COUNT) { + name += "."; + name += k_FileTypeNames[type - WFT_FIRST]; + } + + // Create the entry + Entry entry = createNewEntry(); + entry.offset = offset; + entry.length = size; + entry.unbaked = false; + + auto entryPath = wad->cleanEntryPath(name); + wad->entries.emplace(entryPath, entry); + + if (callback) { + callback(entryPath, entry); + } + } + + return packFile; +} + +// Read and return entry file data from disk or from memory +std::optional> WAD3::readEntry(const std::string& path_) const { + auto path = this->cleanEntryPath(path_); + auto entry = this->findEntry(path); + if (!entry) { + return std::nullopt; + } + + // We have it in memory, no need to read from disk + if (entry->unbaked) { + return readUnbakedEntry(*entry); + } + + // It's baked into the file on disk. Open it, read it, send it back + FileStream stream{this->fullFilePath}; + if (!stream) { + return std::nullopt; + } + stream.seek_in_u(entry->offset); + return stream.read_bytes(entry->length); +} + +void WAD3::addEntryInternal(Entry& entry, const std::string& path, std::vector& buffer, EntryOptions options) { + entry.length = buffer.size(); + + // Offset will be reset when it's baked + entry.offset = 0; +} + +bool WAD3::bake(const std::string& outputDir_, BakeOptions options, const EntryCallback& callback) { + // Get the proper file output folder + std::string outputDir = this->getBakeOutputDir(outputDir_); + std::string outputPath = outputDir + '/' + this->getFilename(); + + // File output stream + FileStream stream{outputPath, FileStream::OPT_TRUNCATE | FileStream::OPT_CREATE_IF_NONEXISTENT}; + stream.seek_out(0); + + // File data to be appended at the end + std::vector fileData; + + // We'll quickly tally up and preallocate our vector so we don't need to do extra allocations at run time + uint32_t totalDataLength = 0; + this->runForAllEntries([&totalDataLength](const std::string&, const Entry& entry) { + totalDataLength += entry.length; + }); + fileData.reserve(totalDataLength); + + // Counter to keep track of where we're writing data to, starts after the header + uint32_t currentOffset = sizeof(uint32_t) * 3; + + // Write out the header + stream.write(WAD3_SIGNATURE); + stream.write(this->getEntryCount()); + stream.write(currentOffset); + + // Push forward to end of lump info array + currentOffset += this->getEntryCount() * 32; + + this->runForAllEntriesInternal([this, &callback, &stream, &fileData, currentOffset](std::string path, Entry& entry) { + // Append the lump data to the end of the data vector + if (auto binData = this->readEntry(path)) { + entry.offset = currentOffset + fileData.size(); + fileData.insert(fileData.end(), binData->begin(), binData->end()); + } + else { + entry.offset = 0; + entry.length = 0; + } + + // Convert the extension back into the type + int type = 0; + std::size_t pos = path.find_last_of('.'); + if (pos > 0) { + std::string_view ext = path.c_str() + pos + 1; + for (int i = WFT_FIRST; i < WFT_COUNT; i++) { + if (string::iequals(ext, k_FileTypeNames[i - WFT_FIRST])) { + type = i; + break; + } + } + // Chop off the extension + path = path.substr(0, pos); + } + + // Write the lump header + stream.write(entry.offset); + stream.write(0); + stream.write(entry.length); + stream.write(type); // type + stream.write(0); // compression + stream.write(0); // padding + stream.write(path, false, WAD3_FILENAME_MAX_SIZE); + + if (callback) { + callback(path, entry); + } + }); + + // File data + stream.write(fileData); + + // Clean up + this->mergeUnbakedEntries(); + PackFile::setFullFilePath(outputDir); + return true; +} + +Attribute WAD3::getSupportedEntryAttributes() const { + using enum Attribute; + return LENGTH; +} From f33774541af0cb918c074fd7b823f2e9a3d15c35 Mon Sep 17 00:00:00 2001 From: craftablescience Date: Tue, 1 Oct 2024 15:29:21 -0400 Subject: [PATCH 2/3] feat(lang): update vpkpp C/C# bindings for WAD3 --- lang/c/include/vpkppc/PackFileType.h | 1 + lang/c/include/vpkppc/format/WAD3.h | 9 +++++ lang/c/src/vpkppc/_vpkppc.cmake | 2 ++ lang/c/src/vpkppc/format/WAD3.cpp | 29 ++++++++++++++++ lang/csharp/src/vpkpp/Format/WAD3.cs | 52 ++++++++++++++++++++++++++++ 5 files changed, 93 insertions(+) create mode 100644 lang/c/include/vpkppc/format/WAD3.h create mode 100644 lang/c/src/vpkppc/format/WAD3.cpp create mode 100644 lang/csharp/src/vpkpp/Format/WAD3.cs diff --git a/lang/c/include/vpkppc/PackFileType.h b/lang/c/include/vpkppc/PackFileType.h index 914f30ae9..5324d3045 100644 --- a/lang/c/include/vpkppc/PackFileType.h +++ b/lang/c/include/vpkppc/PackFileType.h @@ -14,6 +14,7 @@ typedef enum { VPKPP_PACK_FILE_TYPE_PCK, VPKPP_PACK_FILE_TYPE_VPK, VPKPP_PACK_FILE_TYPE_VPK_VTMB, + VPKPP_PACK_FILE_TYPE_WAD3, VPKPP_PACK_FILE_TYPE_ZIP, } vpkpp_pack_file_type_e; diff --git a/lang/c/include/vpkppc/format/WAD3.h b/lang/c/include/vpkppc/format/WAD3.h new file mode 100644 index 000000000..8747cf329 --- /dev/null +++ b/lang/c/include/vpkppc/format/WAD3.h @@ -0,0 +1,9 @@ +#pragma once + +#include "../PackFile.h" + +// REQUIRES MANUAL FREE: vpkpp_close +SOURCEPP_API vpkpp_pack_file_handle_t vpkpp_wad3_create(const char* path); + +// REQUIRES MANUAL FREE: vpkpp_close +SOURCEPP_API vpkpp_pack_file_handle_t vpkpp_wad3_open(const char* path, vpkpp_entry_callback_t callback); diff --git a/lang/c/src/vpkppc/_vpkppc.cmake b/lang/c/src/vpkppc/_vpkppc.cmake index 0f115eb58..c036304c6 100644 --- a/lang/c/src/vpkppc/_vpkppc.cmake +++ b/lang/c/src/vpkppc/_vpkppc.cmake @@ -8,6 +8,7 @@ add_pretty_parser(vpkpp C "${CMAKE_CURRENT_SOURCE_DIR}/lang/c/include/vpkppc/format/PCK.h" "${CMAKE_CURRENT_SOURCE_DIR}/lang/c/include/vpkppc/format/VPK.h" "${CMAKE_CURRENT_SOURCE_DIR}/lang/c/include/vpkppc/format/VPK_VTMB.h" + "${CMAKE_CURRENT_SOURCE_DIR}/lang/c/include/vpkppc/format/WAD3.h" "${CMAKE_CURRENT_SOURCE_DIR}/lang/c/include/vpkppc/format/ZIP.h" "${CMAKE_CURRENT_SOURCE_DIR}/lang/c/include/vpkppc/Attribute.h" "${CMAKE_CURRENT_SOURCE_DIR}/lang/c/include/vpkppc/Entry.h" @@ -25,6 +26,7 @@ add_pretty_parser(vpkpp C "${CMAKE_CURRENT_LIST_DIR}/format/PCK.cpp" "${CMAKE_CURRENT_LIST_DIR}/format/VPK.cpp" "${CMAKE_CURRENT_LIST_DIR}/format/VPK_VTMB.cpp" + "${CMAKE_CURRENT_LIST_DIR}/format/WAD3.cpp" "${CMAKE_CURRENT_LIST_DIR}/format/ZIP.cpp" "${CMAKE_CURRENT_LIST_DIR}/Convert.cpp" "${CMAKE_CURRENT_LIST_DIR}/Entry.cpp" diff --git a/lang/c/src/vpkppc/format/WAD3.cpp b/lang/c/src/vpkppc/format/WAD3.cpp new file mode 100644 index 000000000..f0d8b49d9 --- /dev/null +++ b/lang/c/src/vpkppc/format/WAD3.cpp @@ -0,0 +1,29 @@ +#include + +#include + +#include + +using namespace vpkpp; + +SOURCEPP_API vpkpp_pack_file_handle_t vpkpp_wad3_create(const char* path) { + SOURCEPP_EARLY_RETURN_VAL(path, nullptr); + + auto packFile = WAD3::create(path); + if (!packFile) { + return nullptr; + } + return packFile.release(); +} + +SOURCEPP_API vpkpp_pack_file_handle_t vpkpp_wad3_open(const char* path, vpkpp_entry_callback_t callback) { + SOURCEPP_EARLY_RETURN_VAL(path, nullptr); + + auto packFile = WAD3::open(path, callback ? [callback](const std::string& path, const Entry& entry) { + callback(path.c_str(), const_cast(&entry)); + } : static_cast(nullptr)); + if (!packFile) { + return nullptr; + } + return packFile.release(); +} diff --git a/lang/csharp/src/vpkpp/Format/WAD3.cs b/lang/csharp/src/vpkpp/Format/WAD3.cs new file mode 100644 index 000000000..4cc69de25 --- /dev/null +++ b/lang/csharp/src/vpkpp/Format/WAD3.cs @@ -0,0 +1,52 @@ +using System; +using System.Runtime.InteropServices; + +namespace vpkpp.Format +{ + using EntryCallback = Action; + + internal static unsafe partial class Extern + { + [DllImport("vpkppc")] + public static extern void* vpkpp_wad3_create([MarshalAs(UnmanagedType.LPStr)] string path); + + [DllImport("vpkppc")] + public static extern void* vpkpp_wad3_open([MarshalAs(UnmanagedType.LPStr)] string path, IntPtr callback); + } + + public class WAD3 : PackFile + { + private protected unsafe WAD3(void* handle) : base(handle) {} + + public static WAD3? Create(string path) + { + unsafe + { + var handle = Extern.vpkpp_wad3_create(path); + return handle == null ? null : new WAD3(handle); + } + } + + public new static WAD3? Open(string path) + { + unsafe + { + var handle = Extern.vpkpp_wad3_open(path, 0); + return handle == null ? null : new WAD3(handle); + } + } + + public new static WAD3? Open(string path, EntryCallback callback) + { + unsafe + { + EntryCallbackNative callbackNative = (path, entry) => + { + callback(path, new Entry(entry, true)); + }; + var handle = Extern.vpkpp_wad3_open(path, Marshal.GetFunctionPointerForDelegate(callbackNative)); + return handle == null ? null : new WAD3(handle); + } + } + } +} From d30cfcaa5205c823ed292a946d5e9021692840ca Mon Sep 17 00:00:00 2001 From: craftablescience Date: Tue, 1 Oct 2024 14:47:58 -0400 Subject: [PATCH 3/3] fix(vtfpp): write rgba16, hdr files correctly --- src/vtfpp/ImageConversion.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/vtfpp/ImageConversion.cpp b/src/vtfpp/ImageConversion.cpp index 2c6094111..cac8d6dc0 100644 --- a/src/vtfpp/ImageConversion.cpp +++ b/src/vtfpp/ImageConversion.cpp @@ -882,7 +882,7 @@ std::vector ImageConversion::convertImageDataToFile(std::span ImageConversion::convertImageDataToFile(std::span ImageConversion::convertImageDataToFile(std::span(imageData.data())); + stbi_write_hdr_to_func(stbWriteFunc, &out, width, height, ImageFormatDetails::bpp(ImageFormat::RGBA32323232F) / (8 * sizeof(float)), reinterpret_cast(imageData.data())); } else { auto hdr = convertImageDataToFormat(imageData, format, ImageFormat::RGBA32323232F, width, height); - stbi_write_hdr_to_func(stbWriteFunc, &out, width, height, ImageFormatDetails::bpp(ImageFormat::RGBA32323232F) / 8, reinterpret_cast(hdr.data())); + stbi_write_hdr_to_func(stbWriteFunc, &out, width, height, ImageFormatDetails::bpp(ImageFormat::RGBA32323232F) / (8 * sizeof(float)), reinterpret_cast(hdr.data())); } break; }