From 6cb6d3c16bb8cab8c92656b9192e07d604cbfd46 Mon Sep 17 00:00:00 2001 From: Tholp1 <76782621+tholp1@users.noreply.github.com> Date: Sat, 21 Sep 2024 21:45:09 -0500 Subject: [PATCH 1/6] bsppp: add support for compressed lumps, refactor writing, add entity lump parser --- include/bsppp/LumpData.h | 35 +++ include/bsppp/bsppp.h | 57 ++++- src/bsppp/_bsppp.cmake | 4 +- src/bsppp/bsppp.cpp | 518 ++++++++++++++++++++++++++++++++------- 4 files changed, 520 insertions(+), 94 deletions(-) diff --git a/include/bsppp/LumpData.h b/include/bsppp/LumpData.h index 20af29078..5306aafa9 100644 --- a/include/bsppp/LumpData.h +++ b/include/bsppp/LumpData.h @@ -1,9 +1,30 @@ #pragma once +#include + #include namespace bsppp { +#pragma pack(push) +#pragma pack(1) +// Compressed lumps use their own header to annoy programmers 20 years later +// https://developer.valvesoftware.com/wiki/BSP_(Source)#Lump_compression +struct lzma_header_bsplump +{ + unsigned int id; + unsigned int actualSize; // always little endian + unsigned int lzmaSize; // always little endian + unsigned char properties[5]; +}; + +struct lzma_header_alone // .LZMA format header +{ + unsigned char properties[5]; + unsigned long actualSize; // always little endian +}; +#pragma pack(pop) + //region Lump 1 (Planes) struct BSPPlane_v0 { @@ -177,4 +198,18 @@ using BSPBrushModel = BSPBrushModel_v0; //endregion +//region Lump 35 (Game Lump) +struct BSPGameLump { + int32_t id; + uint16_t flags; + uint16_t version; + int32_t fileoffset;// offset from begining of file, not lump + int32_t length; // (Decompressed) size, compressed size is determined by subtracting the next entry's offset with this one + + + // This being in the struct is not acurate to how it is written to disk, only to manage it here better + std::vector data; +}; + +//endregion } // namespace bsppp diff --git a/include/bsppp/bsppp.h b/include/bsppp/bsppp.h index 4d097c5a9..cc147a01b 100644 --- a/include/bsppp/bsppp.h +++ b/include/bsppp/bsppp.h @@ -6,6 +6,7 @@ #include #include #include +#include #include @@ -15,6 +16,9 @@ namespace bsppp { constexpr auto BSP_SIGNATURE = sourcepp::parser::binary::makeFourCC("VBSP"); +constexpr auto BSP_LZMA_ID = sourcepp::parser::binary::makeFourCC("LZMA"); + +typedef std::map BSPEntity; enum class BSPLump : int32_t { UNKNOWN = -1, @@ -103,8 +107,18 @@ enum class BSPLump : int32_t { }; static_assert(static_cast(BSPLump::COUNT) == 64, "Incorrect lump count!"); +enum BSPGameLumpID +{ + STATIC_PROPS = sourcepp::parser::binary::makeFourCC("sprp"), + DETAIL_PROPS = sourcepp::parser::binary::makeFourCC("dprp"), + DETAIL_PROP_LIGHTING_LDR = sourcepp::parser::binary::makeFourCC("dplt"), + DETAIL_PROP_LIGHTING_HDR = sourcepp::parser::binary::makeFourCC("dplh"), + +}; + constexpr auto BSP_LUMP_COUNT = static_cast(BSPLump::COUNT); +/// Changes made are batched until writeChangesToDisk() is called class BSP { struct Lump { /// Byte offset into file @@ -143,15 +157,20 @@ class BSP { [[nodiscard]] bool hasLump(BSPLump lumpIndex) const; + [[nodiscard]] bool isLumpCompressed(BSPLump lumpIndex) const; + [[nodiscard]] int32_t getLumpVersion(BSPLump lumpIndex) const; void setLumpVersion(BSPLump lumpIndex, int32_t version); - [[nodiscard]] std::optional> readLump(BSPLump lumpIndex) const; + [[nodiscard]] std::optional> readLump(BSPLump lumpIndex, bool readRaw = false, bool readStagedVersion = false) const; + - template - [[nodiscard]] auto readLump() const { - if constexpr (Lump == BSPLump::PLANES) { + template + [[nodiscard]] auto readLump() const { + if constexpr (Lump == BSPLump::ENTITIES){ + return this->parseEntities(); + } else if constexpr (Lump == BSPLump::PLANES) { return this->readPlanes(); } else if constexpr (Lump == BSPLump::TEXDATA) { return this->readTextureData(); @@ -169,20 +188,36 @@ class BSP { return this->readBrushModels(); } else if constexpr (Lump == BSPLump::ORIGINALFACES) { return this->readOriginalFaces(); + } else if constexpr (Lump == BSPLump::GAME_LUMP) { + return this->readGameLumps(); } else { return this->readLump(Lump); } } - void writeLump(BSPLump lumpIndex, std::span data, bool condenseFile = true); + /// stageGameLump Should be used for writing game lumps as they need special handling + /// PAKLUMP can be written here but you should probably use vpkpp for that + /// Valid compressLevel range is 0 to 9, 0 is considered off, 9 the slowest and most compressiest + bool stageLump(BSPLump lumpIndex, std::span data, uint8_t compressLevel = 0); + bool stageLump(std::span, uint8_t compressLevel = 0); + + /// Write all changes made to header and lumps to file + void writeChangesToDisk(); + /// Reset changes made to a lump to how it is on disk currently, not specicfying one resets all + void flushChanges(BSPLump lumpIndex = BSPLump::UNKNOWN); - bool applyLumpPatchFile(const std::string& lumpFilePath); + bool stageLumpPatchFile(const std::string& lumpFilePath); void createLumpPatchFile(BSPLump lumpIndex) const; + bool stageGameLump(char id[4], uint16_t flags, uint16_t version, std::span data, uint8_t compressLevel); + bool stageGameLump(int32_t id, uint16_t flags, uint16_t version, std::span data, uint8_t compressLevel); + protected: void writeHeader() const; + [[nodiscard]] std::vector parseEntities() const; + [[nodiscard]] std::vector readPlanes() const; [[nodiscard]] std::vector readTextureData() const; @@ -201,8 +236,18 @@ class BSP { [[nodiscard]] std::vector readOriginalFaces() const; + [[nodiscard]] std::vector readGameLumps() const; + + [[nodiscard]] static std::optional> compressLumpData(const std::span data, uint8_t compressLevel = 6); + + [[nodiscard]] static std::optional> decompressLumpData(const std::span data); + std::string path; Header header{}; + Header stagedHeader{}; + + std::array, BSP_LUMP_COUNT> stagedLumps; + std::vector stagedGameLumps; // Slightly different header just to be quirky and special bool isL4D2 = false; diff --git a/src/bsppp/_bsppp.cmake b/src/bsppp/_bsppp.cmake index f3fc447a0..08d2dbd6f 100644 --- a/src/bsppp/_bsppp.cmake +++ b/src/bsppp/_bsppp.cmake @@ -1,5 +1,5 @@ add_pretty_parser(bsppp - DEPS MINIZIP::minizip sourcepp_parser sourcepp::vpkpp + DEPS liblzma MINIZIP::minizip sourcepp_parser sourcepp::vpkpp PRECOMPILED_HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/include/bsppp/bsppp.h" "${CMAKE_CURRENT_SOURCE_DIR}/include/bsppp/LumpData.h" @@ -7,3 +7,5 @@ add_pretty_parser(bsppp SOURCES "${CMAKE_CURRENT_LIST_DIR}/bsppp.cpp" "${CMAKE_CURRENT_LIST_DIR}/PakLump.cpp") + +target_include_directories(bsppp PRIVATE "${xz_SOURCE_DIR}/src/liblzma/api") diff --git a/src/bsppp/bsppp.cpp b/src/bsppp/bsppp.cpp index df0566558..c74dc4946 100644 --- a/src/bsppp/bsppp.cpp +++ b/src/bsppp/bsppp.cpp @@ -1,11 +1,12 @@ #include +#include #include #include #include -#include #include +#include using namespace bsppp; using namespace sourcepp; @@ -81,6 +82,8 @@ BSP::BSP(std::string path_) } reader >> this->header.mapRevision; + this->stagedHeader = this->header; + this->stagedGameLumps = this->readGameLumps(); } BSP::operator bool() const { @@ -104,8 +107,7 @@ int32_t BSP::getVersion() const { } void BSP::setVersion(int32_t version) { - this->header.version = version; - this->writeHeader(); + this->stagedHeader.version = version; } int32_t BSP::getMapRevision() const { @@ -113,18 +115,24 @@ int32_t BSP::getMapRevision() const { } void BSP::setMapRevision(int32_t mapRevision) { - this->header.mapRevision = mapRevision; - this->writeHeader(); + this->stagedHeader.mapRevision = mapRevision; } bool BSP::hasLump(BSPLump lumpIndex) const { if (this->path.empty()) { return false; } - const auto lump = static_cast>(lumpIndex); + const auto lump = static_cast>(lumpIndex); return this->header.lumps[lump].length != 0 && this->header.lumps[lump].offset != 0; } +bool BSP::isLumpCompressed(BSPLump lumpIndex) const { + if (this->hasLump(lumpIndex)){ + return this->header.lumps[static_cast>(lumpIndex)].fourCC > 0; + } + return false; +} + int32_t BSP::getLumpVersion(BSPLump lumpIndex) const { if (this->path.empty()) { return 0; @@ -133,106 +141,186 @@ int32_t BSP::getLumpVersion(BSPLump lumpIndex) const { } void BSP::setLumpVersion(BSPLump lumpIndex, int32_t version) { - this->header.lumps[static_cast>(lumpIndex)].version = version; - this->writeHeader(); + this->stagedHeader.lumps[static_cast>(lumpIndex)].version = version; } -std::optional> BSP::readLump(BSPLump lumpIndex) const { - if (this->path.empty() || !this->hasLump(lumpIndex)) { +std::optional> BSP::readLump(BSPLump lumpIndex, bool readRaw, bool stagedVersion) const { + if (this->path.empty() || !this->hasLump(lumpIndex)) { return std::nullopt; } - FileStream reader{this->path}; - return reader - .seek_in(this->header.lumps[static_cast>(lumpIndex)].offset) - .read_bytes(this->header.lumps[static_cast>(lumpIndex)].length); + + auto lumpToRead = static_cast>(lumpIndex); + std::vector lumpBytes; + + if (stagedVersion) { + lumpBytes = this->stagedLumps[lumpToRead]; + } + else{ + FileStream reader{this->path}; + lumpBytes = reader + .seek_in(this->header.lumps[lumpToRead].offset) + .read_bytes(this->header.lumps[lumpToRead].length); + } + + if ( this->isLumpCompressed(lumpIndex) && !readRaw) { + return decompressLumpData(lumpBytes); + } + else { + // Not Compressed and we dont need to do anything + return lumpBytes; + } + } -void BSP::writeLump(BSPLump lumpIndex, std::span data, bool condenseFile) { +bool BSP::stageLump(BSPLump lumpIndex, const std::span data, uint8_t compressLevel) { + if (this->path.empty() || lumpIndex == BSPLump::UNKNOWN) { - return; + return false; } - auto lumpToMove = static_cast>(lumpIndex); + auto lumpToStage = static_cast>(lumpIndex); - if (!this->hasLump(lumpIndex) || !condenseFile) { - // Put the lump at the end of the file - int32_t lastLumpOffset = 0, lastLumpLength = 0; - for (const auto& [lumpOffset, lumpLength, lumpVersion, lumpFourCC] : this->header.lumps) { - if (lastLumpOffset < this->header.lumps[lumpToMove].offset) { - lastLumpOffset = lumpOffset; - lastLumpLength = lumpLength; - } - } - if (lastLumpOffset == 0) { - // Whole file is full of empty lumps - this->header.lumps[lumpToMove].offset = sizeof(Header); - } else { - this->header.lumps[lumpToMove].offset = lastLumpOffset + lastLumpLength; - } - this->header.lumps[lumpToMove].length = static_cast(data.size()); - } else { - // Sort lumps by file position - std::array lumpIDs{}; - for (int i = 0; i < lumpIDs.size(); i++) { - lumpIDs[i] = i; - } - std::sort(lumpIDs.begin(), lumpIDs.end(), [this](int lhs, int rhs) { - return this->header.lumps[lhs].offset < this->header.lumps[rhs].offset; - }); - - // Condense the lumps in the order they appear in the file, and move the new lump to the end - FileStream bsp{this->path, FileStream::OPT_READ | FileStream::OPT_WRITE}; - int32_t currentOffset = 0; - for (int i = 0; i < lumpIDs.size(); i++) { - auto lumpID = lumpIDs[i]; - - if (lumpID == lumpToMove) { - continue; - } - if (!currentOffset) { - currentOffset = this->header.lumps[lumpID].offset + this->header.lumps[lumpID].length; - continue; - } + if (compressLevel > 0) { + auto newLump = compressLumpData(data, compressLevel); + if (!newLump.has_value()) + return false; - auto lumpsData = bsp.seek_in(this->header.lumps[lumpID].offset).read_bytes(this->header.lumps[lumpID].length); - bsp.seek_out(currentOffset).write(lumpsData); + this->stagedHeader.lumps[lumpToStage].fourCC = data.size(); + this->stagedLumps[lumpToStage].clear(); + this->stagedLumps[lumpToStage].insert(stagedLumps[lumpToStage].begin(), newLump.value().begin(), newLump.value().end()); + } + else { + this->stagedHeader.lumps[lumpToStage].fourCC = 0; + this->stagedLumps[lumpToStage].clear(); + this->stagedLumps[lumpToStage].insert(stagedLumps[lumpToStage].begin(), data.begin(), data.end()); + } - this->header.lumps[lumpID].offset = currentOffset; - currentOffset += this->header.lumps[lumpID].length; - - // If we have the space to add padding (we should), then do so - // This should never fail for well-constructed BSP files - auto padding = math::paddingForAlignment(4, currentOffset); - if (padding && i < lumpIDs.size() - 1 && currentOffset + padding <= this->header.lumps[lumpIDs[i + 1]].offset) { - currentOffset += padding; - } - } + return true; +} - this->header.lumps[lumpToMove].offset = currentOffset; - this->header.lumps[lumpToMove].length = static_cast(data.size()); - } +bool BSP::stageLump(std::span entities, uint8_t compressLevel) { + if (this->path.empty()) { + return false; + } + + std::vector byteVec; + for (auto e : entities) + { + byteVec.push_back((std::byte)'{'); + for (auto it = e.begin(); it != e.end(); it++) { + byteVec.push_back((std::byte)'\"'); + + for (char c : it->first){ + byteVec.push_back((std::byte)c); + } + byteVec.push_back((std::byte)'\"'); + byteVec.push_back((std::byte)' '); + byteVec.push_back((std::byte)'\"'); + for (char c : it->second){ + byteVec.push_back((std::byte)c); + } + byteVec.push_back((std::byte)'\"'); + } + byteVec.push_back((std::byte)'}'); + } + return this->stageLump(BSPLump::ENTITIES, byteVec, compressLevel); +} - // Write modified header and lump - this->writeHeader(); - { - FileStream writer{this->path, FileStream::OPT_READ | FileStream::OPT_WRITE}; - writer.seek_out(this->header.lumps[lumpToMove].offset).write(data); - } +void BSP::writeChangesToDisk() { + uint32_t offset = sizeof(BSP_SIGNATURE) + sizeof(BSP::Header); + this->header = this->stagedHeader; + + for ( int32_t i = 0; i < BSP_LUMP_COUNT; i++){ // Fill empty space with lumps we're keeping + if (i == static_cast(BSPLump::GAME_LUMP) + && this->stagedGameLumps.size() > 0) {// Need special handling since its split up + + uint32_t gameLumpOffset = offset; + if (this->stagedGameLumps[0].data.size() != this->stagedGameLumps[0].length) { // + // length member is decompressed size, need to add an empty item with an offset so the compressed size is known later + BSPGameLump dummy{0,0,0,0,0}; + this->stagedGameLumps.push_back(dummy); + } + + auto &lump = this->stagedLumps[i]; + + lump.clear(); + BufferStream stream(lump); + stream.write(this->stagedGameLumps.size()); + gameLumpOffset += sizeof(int32_t); + gameLumpOffset += (sizeof(BSPGameLump) - sizeof(BSPGameLump::data)) * this->stagedGameLumps.size(); + //gameLumpOffset += 128 * this->stagedGameLumps.size(); + + for (auto gameLump : this->stagedGameLumps) { + if (gameLump.id == 0) { + gameLump.fileoffset = 0; + } + else { + gameLump.fileoffset = gameLumpOffset; + gameLumpOffset += gameLump.data.size(); + } + stream.write(gameLump.id) + .write(gameLump.flags) + .write(gameLump.version) + .write(gameLump.fileoffset) + .write(gameLump.length); + } + for (auto gameLump : this->stagedGameLumps) + { + if (gameLump.id == 0) + continue; + stream.write(gameLump.data); + } + this->header.lumps[i].offset = offset; + this->header.lumps[i].length = lump.size(); + offset += lump.size(); + + } + else { + auto &lump = this->stagedLumps[i]; + if (lump.size() == 0){ + if (this->hasLump(static_cast(i))) + lump = this->readLump(static_cast(i), true).value(); + } + + if (lump.size() == 0) + this->header.lumps[i].offset = 0; + else{ + this->header.lumps[i].offset = offset; + offset += lump.size(); + } + this->header.lumps[i].length = lump.size(); + } + } + + this->writeHeader(); + FileStream writer{ this->path, FileStream::OPT_READ | FileStream::OPT_WRITE }; + + for ( int i = 0; i < BSP_LUMP_COUNT; i++){ + if (this->stagedLumps[i].size()) + writer.seek_out(this->header.lumps[i].offset) + .write(this->stagedLumps[i]); + } +} - // Resize file if it shrank - int32_t lastLumpOffset = 0, lastLumpLength = 0; - for (const auto& [lumpOffset, lumpLength, lumpVersion, lumpFourCC] : this->header.lumps) { - if (lastLumpOffset < this->header.lumps[lumpToMove].offset) { - lastLumpOffset = lumpOffset; - lastLumpLength = lumpLength; +void BSP::flushChanges(BSPLump lumpIndex) +{ + if (lumpIndex == BSPLump::UNKNOWN) { + for (int i = 0; i < static_cast>(BSPLump::COUNT); i++) { + this->flushChanges(static_cast(i)); } + return; } - if (std::filesystem::file_size(this->path) > lastLumpOffset + lastLumpLength) { - std::filesystem::resize_file(this->path, lastLumpOffset + lastLumpLength); + + + if (lumpIndex == BSPLump::GAME_LUMP) { + this->stagedGameLumps = this->readGameLumps(); } + auto lumpToFlush = static_cast>(lumpIndex); + this->stagedLumps[lumpToFlush].clear(); + this->stagedHeader.lumps[lumpToFlush] = this->header.lumps[lumpToFlush]; } -bool BSP::applyLumpPatchFile(const std::string& lumpFilePath) { +bool BSP::stageLumpPatchFile(const std::string& lumpFilePath) { if (this->path.empty()) { return false; } @@ -251,7 +339,7 @@ bool BSP::applyLumpPatchFile(const std::string& lumpFilePath) { } this->header.lumps[index].version = version; - this->writeLump(static_cast(index), reader.seek_in(offset).read_bytes(length)); + this->stageLump(static_cast(index), reader.seek_in(offset).read_bytes(length)); return true; } @@ -283,6 +371,31 @@ void BSP::createLumpPatchFile(BSPLump lumpIndex) const { .write(*lumpData); } +bool BSP::stageGameLump(char id[4], uint16_t flags, uint16_t version, std::span data, uint8_t compressLevel) { + return this->stageGameLump(id[0] | (id[1] << 8) | (id[2] << 16) | (id[3] << 24) + , flags, version, data, compressLevel); +} + +bool BSP::stageGameLump(int32_t id, uint16_t flags, uint16_t version, std::span data, uint8_t compressLevel) { + BSPGameLump gameLump; + gameLump.id = id; + gameLump.flags = flags; + gameLump.version = version; + gameLump.length = data.size(); + + if (compressLevel){ + auto compressed = compressLumpData(data, compressLevel); + if (!compressed.has_value()) + return false; + gameLump.data = compressed.value(); + } else { + gameLump.data.clear(); + gameLump.data.insert(gameLump.data.begin(), data.begin(), data.end()); + } + this->stagedGameLumps.push_back(gameLump); + return true; +} + void BSP::writeHeader() const { FileStream writer{this->path, FileStream::OPT_READ | FileStream::OPT_WRITE}; writer.seek_out(0) << BSP_SIGNATURE << this->header.version; @@ -303,6 +416,66 @@ void BSP::writeHeader() const { writer << this->header.mapRevision; } +std::vector BSP::parseEntities() const { + auto lumpData = this->readLump(BSPLump::ENTITIES); + + std::vector ents; + + if (!lumpData.has_value()) + { + return ents; + } + + auto lumpDataValue = lumpData.value(); + + BSPEntity e; + std::string key, value; + bool quoted = false; + bool keying = true; + + for (uint32_t i = 0; i BSP::readPlanes() const { return ::readLumpContents(*this, BSPLump::PLANES); } @@ -357,3 +530,174 @@ std::vector BSP::readOriginalFaces() const { } }); } + +std::vector BSP::readGameLumps() const { + std::vector lumps; + + auto lumpsAggregate = this->readLump(BSPLump::GAME_LUMP, true); + if (!lumpsAggregate.has_value()) + return lumps; + + BufferStreamReadOnly stream{lumpsAggregate.value()}; + + uint32_t numGameLumps = stream.read(); + for (int i = 0; i < numGameLumps; i++) + { + BSPGameLump lump; + stream + .read(lump.id) + .read(lump.flags) + .read(lump.version) + .read(lump.fileoffset) + .read(lump.length); + // if (lump.id != 0) + lumps.push_back(lump); + } + bool isCompressed = lumps.back().id == 0; // Game Lumps are compressed individually, + //when they are there is an empty entry with only the offset to calculate compressed size + + for (int i = 0; i < numGameLumps; i++) { + if (lumps[i].id == 0) + break; + if (!isCompressed) { + lumps[i].data = stream.read_bytes(lumps[i].length); + } else { + //printf("%i: ", i); + int32_t nextOffset = lumps[i+1].fileoffset; + if (nextOffset == 0) { + auto id = static_cast>(BSPLump::GAME_LUMP); + nextOffset = this->header.lumps[id].offset + this->header.lumps[id].length; + } + int32_t length = nextOffset - lumps[i].fileoffset; + //printf("%i\n", length); + auto data = stream.read_bytes(length); + auto decompressed = decompressLumpData(data); + + if (!decompressed.has_value()) + lumps[i].data = data; + else + lumps[i].data = decompressed.value(); + } + } + if (lumps.back().id == 0) + lumps.pop_back(); + + return lumps; + +} + +std::optional> BSP::compressLumpData(const std::span data, uint8_t compressLevel){ + uint32_t bufsize = data.size_bytes() + sizeof(lzma_header_alone); + std::byte *compressedData = (std::byte*)malloc(bufsize); + lzma_stream stream = LZMA_STREAM_INIT; + + stream.next_in = (uint8_t*)data.data(); + stream.avail_in = data.size_bytes(); // + 32; + stream.next_out = (uint8_t*)compressedData; + stream.avail_out = bufsize; + + lzma_options_lzma options; + if ( compressLevel > 9 )// Out of allowed range + compressLevel = 6; + lzma_lzma_preset(&options, compressLevel); + lzma_ret initError = lzma_alone_encoder(&stream, &options); + + if (initError) { + lzma_end(&stream); + //printf("init %i", initError); + free(compressedData); + return std::nullopt; + } + + lzma_ret compressError; + while (true){//(compressError == LZMA_OK) { + compressError = lzma_code(&stream, LZMA_RUN); + // if (stream.total_in >= data.size()) { + // break; + // } + if (compressError != LZMA_OK) + { + break; + } + } + // printf("%i\n", compressError); + //if (compressError == LZMA_STREAM_END) + //{ + compressError = lzma_code(&stream, LZMA_RUN); + compressError = lzma_code(&stream, LZMA_FINISH); + lzma_end(&stream); + //} + + // if (compressError != LZMA_OK) { + // //printf("compress %i\n", compressError); + // lzma_end(&stream); + // return false; + // } + + std::vector compressedLump; + //newLump.insert(newLump.begin(), compressedData, compressedData + stream.total_out); + + // Transplant header with Valve's + lzma_header_bsplump headerBSP; + lzma_header_alone *headerAlone = (lzma_header_alone*)compressedData; + + headerBSP.id = BSP_LZMA_ID; + headerBSP.actualSize = stream.total_in; + headerBSP.lzmaSize = stream.total_out - sizeof(lzma_header_alone); + + //printf("actual: %i\nbsp: %i\n", headerAlone->actualSize, headerBSP.actualSize); + + std::copy(headerAlone->properties, headerAlone->properties + sizeof(headerAlone->properties), headerBSP.properties); + + //newLump.erase(newLump.begin(), newLump.begin() + sizeof(lzma_header_alone)); + //newLump.insert(newLump.begin(), (std::byte*)&headerBSP, ((std::byte*)&headerBSP) + sizeof(lzma_header_bsplump)); + compressedLump.insert(compressedLump.begin(), (std::byte*)&headerBSP, (std::byte*)&headerBSP + sizeof(headerBSP)); + compressedLump.insert(compressedLump.end(), compressedData + sizeof(lzma_header_alone), compressedData + stream.total_out); + free(compressedData); + return compressedLump; +} + +std::optional> BSP::decompressLumpData(const std::span data) { + // Transplant Valve header with a normal one + lzma_header_bsplump *headerBSP = (lzma_header_bsplump*)(data.data()); + lzma_header_alone headerAlone; + headerAlone.actualSize = headerBSP->actualSize; + + std::copy(headerBSP->properties, headerBSP->properties + sizeof(headerBSP->properties), headerAlone.properties); + int32_t uncompressedSize = headerAlone.actualSize; + //data.erase(data.begin(), data.begin() + sizeof(lzma_header_bsplump)); + //data.insert(data.begin(), (std::byte*)&headerAlone, ((std::byte*)&headerAlone) + sizeof(lzma_header_alone)); + std::span fixedHeaderData = data.subspan(sizeof(lzma_header_bsplump) - sizeof(lzma_header_alone), std::dynamic_extent); + *((lzma_header_alone*)fixedHeaderData.data()) = headerAlone; + + std::byte *decodedLump = (std::byte*)malloc(uncompressedSize); + + uint64_t memlimit = 1074000000 ; // 1 GB should be more than enough + lzma_stream stream = LZMA_STREAM_INIT; + stream.next_in = (uint8_t*)fixedHeaderData.data(); + stream.avail_in = fixedHeaderData.size(); + stream.next_out = (uint8_t*)decodedLump; + stream.avail_out = uncompressedSize; + lzma_ret initError = lzma_alone_decoder(&stream, memlimit); + + if (initError) { + //printf("init error %i\n", initError); + lzma_end(&stream); + free(decodedLump); + return std::nullopt; + } + + lzma_ret decompressError; + decompressError = lzma_code(&stream, LZMA_RUN); + decompressError = lzma_code(&stream, LZMA_FINISH); + lzma_end(&stream); + + if (decompressError != LZMA_STREAM_END && decompressError != LZMA_OK) { + return std::nullopt; + } + + std::vector decodedLumpVec; + decodedLumpVec.insert(decodedLumpVec.begin(), decodedLump, decodedLump + uncompressedSize); + free(decodedLump); + return decodedLumpVec; +} From 97965ac773ab4d5e430f1753261c7e9814d1eb97 Mon Sep 17 00:00:00 2001 From: craftablescience Date: Tue, 31 Dec 2024 21:35:36 -0500 Subject: [PATCH 2/6] bsppp: do some refactoring --- include/bsppp/LumpData.h | 91 ++-- include/bsppp/bsppp.h | 153 +++---- src/bsppp/PakLump.cpp | 5 +- src/bsppp/bsppp.cpp | 967 ++++++++++++++++++++------------------- 4 files changed, 630 insertions(+), 586 deletions(-) diff --git a/include/bsppp/LumpData.h b/include/bsppp/LumpData.h index 5306aafa9..7c4b6acfa 100644 --- a/include/bsppp/LumpData.h +++ b/include/bsppp/LumpData.h @@ -2,29 +2,11 @@ #include +#include #include namespace bsppp { -#pragma pack(push) -#pragma pack(1) -// Compressed lumps use their own header to annoy programmers 20 years later -// https://developer.valvesoftware.com/wiki/BSP_(Source)#Lump_compression -struct lzma_header_bsplump -{ - unsigned int id; - unsigned int actualSize; // always little endian - unsigned int lzmaSize; // always little endian - unsigned char properties[5]; -}; - -struct lzma_header_alone // .LZMA format header -{ - unsigned char properties[5]; - unsigned long actualSize; // always little endian -}; -#pragma pack(pop) - //region Lump 1 (Planes) struct BSPPlane_v0 { @@ -62,6 +44,56 @@ using BSPVertex = BSPVertex_v0; //endregion +//region Lump 5 (Nodes) + +struct BSPNode_v0 { + uint32_t planeNum; + int32_t children[2]; + int16_t mins[3]; + int16_t maxs[3]; + uint16_t firstFace; + uint16_t numFaces; + int16_t area; + uint16_t _unused0; +}; + +struct BSPNode_v1 { + uint32_t planeNum; + int32_t children[2]; + float mins[3]; + float maxs[3]; + uint32_t firstFace; + uint32_t numFaces; + int16_t area; + + [[nodiscard]] static BSPNode_v1 upgrade(const BSPNode_v0& old) { + return { + old.planeNum, + { + old.children[0], + old.children[1], + }, + { + static_cast(old.mins[0]), + static_cast(old.mins[1]), + static_cast(old.mins[2]), + }, + { + static_cast(old.maxs[0]), + static_cast(old.maxs[1]), + static_cast(old.maxs[2]), + }, + old.firstFace, + old.numFaces, + old.area, + }; + } +}; + +using BSPNode = BSPNode_v1; + +//endregion + //region Lump 6 (Texture Info) struct BSPTextureInfo_v0 { @@ -199,17 +231,24 @@ using BSPBrushModel = BSPBrushModel_v0; //endregion //region Lump 35 (Game Lump) -struct BSPGameLump { - int32_t id; - uint16_t flags; - uint16_t version; - int32_t fileoffset;// offset from begining of file, not lump - int32_t length; // (Decompressed) size, compressed size is determined by subtracting the next entry's offset with this one +struct BSPGameLump { + enum Signature : uint32_t { + SIGNATURE_STATIC_PROPS = sourcepp::parser::binary::makeFourCC("sprp"), + SIGNATURE_DETAIL_PROPS = sourcepp::parser::binary::makeFourCC("dprp"), + SIGNATURE_DETAIL_PROP_LIGHTING_LDR = sourcepp::parser::binary::makeFourCC("dplt"), + SIGNATURE_DETAIL_PROP_LIGHTING_HDR = sourcepp::parser::binary::makeFourCC("dplh"), + } signature; + uint16_t isCompressed; + uint16_t version; + // Offset from beginning of file, not game lump (except Portal 2 on console but we'll ignore that for now) + uint32_t offset; + // Compressed size is determined by subtracting the next entry's offset with this one like VTF resources + uint32_t uncompressedLength; - // This being in the struct is not acurate to how it is written to disk, only to manage it here better std::vector data; }; //endregion + } // namespace bsppp diff --git a/include/bsppp/bsppp.h b/include/bsppp/bsppp.h index cc147a01b..7df7bb59c 100644 --- a/include/bsppp/bsppp.h +++ b/include/bsppp/bsppp.h @@ -5,8 +5,9 @@ #include #include #include +#include #include -#include +#include #include @@ -16,9 +17,7 @@ namespace bsppp { constexpr auto BSP_SIGNATURE = sourcepp::parser::binary::makeFourCC("VBSP"); -constexpr auto BSP_LZMA_ID = sourcepp::parser::binary::makeFourCC("LZMA"); - -typedef std::map BSPEntity; +constexpr auto LZMA_VALVE_SIGNATURE = sourcepp::parser::binary::makeFourCC("LZMA"); enum class BSPLump : int32_t { UNKNOWN = -1, @@ -105,138 +104,131 @@ enum class BSPLump : int32_t { COUNT, }; -static_assert(static_cast(BSPLump::COUNT) == 64, "Incorrect lump count!"); - -enum BSPGameLumpID -{ - STATIC_PROPS = sourcepp::parser::binary::makeFourCC("sprp"), - DETAIL_PROPS = sourcepp::parser::binary::makeFourCC("dprp"), - DETAIL_PROP_LIGHTING_LDR = sourcepp::parser::binary::makeFourCC("dplt"), - DETAIL_PROP_LIGHTING_HDR = sourcepp::parser::binary::makeFourCC("dplh"), - -}; - -constexpr auto BSP_LUMP_COUNT = static_cast(BSPLump::COUNT); +constexpr int32_t BSP_LUMP_COUNT = 64; +static_assert(static_cast>(BSPLump::COUNT) == BSP_LUMP_COUNT, "Incorrect lump count!"); -/// Changes made are batched until writeChangesToDisk() is called class BSP { struct Lump { /// Byte offset into file - int32_t offset; + uint32_t offset; /// Length of lump data - int32_t length; + uint32_t length; /// Lump format version - int32_t version; - /// Uncompressed size, or 0 - int32_t fourCC; + uint32_t version; + /// Uncompressed length if lump is compressed, else 0 + uint32_t uncompressedLength; }; struct Header { /// Version of the BSP file - int32_t version; + uint32_t version; /// Lump metadata std::array lumps; /// Map version number - int32_t mapRevision; + uint32_t mapRevision; }; public: - explicit BSP(std::string path_); + explicit BSP(std::string path_, bool loadPatchFiles = true); explicit operator bool() const; - static BSP create(std::string path, int32_t version = 21, int32_t mapRevision = 0); + static BSP create(std::string path, uint32_t version = 21, uint32_t mapRevision = 0); - [[nodiscard]] int32_t getVersion() const; + [[nodiscard]] uint32_t getVersion() const; - void setVersion(int32_t version); + void setVersion(uint32_t version); - [[nodiscard]] int32_t getMapRevision() const; + [[nodiscard]] uint32_t getMapRevision() const; - void setMapRevision(int32_t mapRevision); + void setMapRevision(uint32_t mapRevision); [[nodiscard]] bool hasLump(BSPLump lumpIndex) const; - [[nodiscard]] bool isLumpCompressed(BSPLump lumpIndex) const; - - [[nodiscard]] int32_t getLumpVersion(BSPLump lumpIndex) const; + [[nodiscard]] bool isLumpCompressed(BSPLump lumpIndex) const; - void setLumpVersion(BSPLump lumpIndex, int32_t version); + [[nodiscard]] uint32_t getLumpVersion(BSPLump lumpIndex) const; - [[nodiscard]] std::optional> readLump(BSPLump lumpIndex, bool readRaw = false, bool readStagedVersion = false) const; + [[nodiscard]] std::optional> getLumpData(BSPLump lumpIndex, bool noDecompression = false) const; - - template - [[nodiscard]] auto readLump() const { - if constexpr (Lump == BSPLump::ENTITIES){ - return this->parseEntities(); - } else if constexpr (Lump == BSPLump::PLANES) { - return this->readPlanes(); + template + [[nodiscard]] auto getLumpData() const { + if constexpr (Lump == BSPLump::PLANES) { + return this->parsePlanes(); } else if constexpr (Lump == BSPLump::TEXDATA) { - return this->readTextureData(); + return this->parseTextureData(); } else if constexpr (Lump == BSPLump::VERTEXES) { - return this->readVertices(); + return this->parseVertices(); + } else if constexpr (Lump == BSPLump::NODES) { + return this->parseNodes(); } else if constexpr (Lump == BSPLump::TEXINFO) { - return this->readTextureInfo(); + return this->parseTextureInfo(); } else if constexpr (Lump == BSPLump::FACES) { - return this->readFaces(); + return this->parseFaces(); } else if constexpr (Lump == BSPLump::EDGES) { - return this->readEdges(); + return this->parseEdges(); } else if constexpr (Lump == BSPLump::SURFEDGES) { - return this->readSurfEdges(); + return this->parseSurfEdges(); } else if constexpr (Lump == BSPLump::MODELS) { - return this->readBrushModels(); + return this->parseBrushModels(); } else if constexpr (Lump == BSPLump::ORIGINALFACES) { - return this->readOriginalFaces(); + return this->parseOriginalFaces(); } else if constexpr (Lump == BSPLump::GAME_LUMP) { - return this->readGameLumps(); + return this->parseGameLumps(true); } else { - return this->readLump(Lump); + return this->getLumpData(Lump); } } - /// stageGameLump Should be used for writing game lumps as they need special handling - /// PAKLUMP can be written here but you should probably use vpkpp for that - /// Valid compressLevel range is 0 to 9, 0 is considered off, 9 the slowest and most compressiest - bool stageLump(BSPLump lumpIndex, std::span data, uint8_t compressLevel = 0); - bool stageLump(std::span, uint8_t compressLevel = 0); + /// BSP::setGameLump should be used for writing game lumps as they need special handling. Paklump can be + /// written here but compression is unsupported, prefer using bsppp::PakLump or your favorite zip library + /// instead. Valid compressLevel range is 0 to 9, 0 is considered off, 9 the slowest and most compressiest + bool setLump(BSPLump lumpIndex, uint32_t version, std::span data, uint8_t compressLevel = 0); + + [[nodiscard]] bool isGameLumpCompressed(BSPGameLump::Signature signature) const; + + [[nodiscard]] uint16_t getGameLumpVersion(BSPGameLump::Signature signature); - /// Write all changes made to header and lumps to file - void writeChangesToDisk(); - /// Reset changes made to a lump to how it is on disk currently, not specicfying one resets all - void flushChanges(BSPLump lumpIndex = BSPLump::UNKNOWN); + [[nodiscard]] std::optional> getGameLumpData(BSPGameLump::Signature signature) const; - bool stageLumpPatchFile(const std::string& lumpFilePath); + bool setGameLump(BSPGameLump::Signature signature, uint16_t version, std::span data, uint8_t compressLevel = 0); + + /// Reset changes made to a lump before they're written to disk + void resetLump(BSPLump lumpIndex); + + /// Resets ALL in-memory modifications (version, all lumps including game lumps, map revision) + void reset(); void createLumpPatchFile(BSPLump lumpIndex) const; - bool stageGameLump(char id[4], uint16_t flags, uint16_t version, std::span data, uint8_t compressLevel); - bool stageGameLump(int32_t id, uint16_t flags, uint16_t version, std::span data, uint8_t compressLevel); + bool setLumpFromPatchFile(const std::string& lumpFilePath); + + bool bake(std::string_view outputPath = ""); protected: - void writeHeader() const; + bool readHeader(); - [[nodiscard]] std::vector parseEntities() const; + [[nodiscard]] std::vector parsePlanes() const; - [[nodiscard]] std::vector readPlanes() const; + [[nodiscard]] std::vector parseTextureData() const; - [[nodiscard]] std::vector readTextureData() const; + [[nodiscard]] std::vector parseVertices() const; - [[nodiscard]] std::vector readVertices() const; + [[nodiscard]] std::vector parseNodes() const; - [[nodiscard]] std::vector readTextureInfo() const; + [[nodiscard]] std::vector parseTextureInfo() const; - [[nodiscard]] std::vector readFaces() const; + [[nodiscard]] std::vector parseFaces() const; - [[nodiscard]] std::vector readEdges() const; + [[nodiscard]] std::vector parseEdges() const; - [[nodiscard]] std::vector readSurfEdges() const; + [[nodiscard]] std::vector parseSurfEdges() const; - [[nodiscard]] std::vector readBrushModels() const; + [[nodiscard]] std::vector parseBrushModels() const; - [[nodiscard]] std::vector readOriginalFaces() const; + [[nodiscard]] std::vector parseOriginalFaces() const; - [[nodiscard]] std::vector readGameLumps() const; + [[nodiscard]] std::vector parseGameLumps(bool decompress) const; [[nodiscard]] static std::optional> compressLumpData(const std::span data, uint8_t compressLevel = 6); @@ -244,12 +236,13 @@ class BSP { std::string path; Header header{}; - Header stagedHeader{}; - std::array, BSP_LUMP_COUNT> stagedLumps; - std::vector stagedGameLumps; + uint32_t stagedVersion{}; + std::unordered_map>> stagedLumps; + std::vector stagedGameLumps; + uint32_t stagedMapRevision{}; - // Slightly different header just to be quirky and special + // Slightly different header despite using the same version just to be quirky bool isL4D2 = false; }; diff --git a/src/bsppp/PakLump.cpp b/src/bsppp/PakLump.cpp index efbc95df8..ed29ca483 100644 --- a/src/bsppp/PakLump.cpp +++ b/src/bsppp/PakLump.cpp @@ -42,7 +42,7 @@ std::unique_ptr PakLump::open(const std::string& path, const EntryCall bsp->version = reader.getVersion(); bsp->mapRevision = reader.getMapRevision(); - if (auto pakFileLump = reader.readLump(BSPLump::PAKFILE)) { + if (auto pakFileLump = reader.getLumpData(BSPLump::PAKFILE)) { // Extract the paklump to a temp file FileStream writer{bsp->tempPakLumpPath, FileStream::OPT_TRUNCATE | FileStream::OPT_CREATE_IF_NONEXISTENT}; writer.write(*pakFileLump); @@ -111,7 +111,8 @@ bool PakLump::bake(const std::string& outputDir_, BakeOptions options, const Ent if (!writer) { return false; } - writer.writeLump(BSPLump::PAKFILE, fs::readFileBuffer(this->tempZIPPath), false); + writer.setLump(BSPLump::PAKFILE, 0, fs::readFileBuffer(this->tempZIPPath)); + writer.bake(); } // Rename and reopen the ZIP diff --git a/src/bsppp/bsppp.cpp b/src/bsppp/bsppp.cpp index c74dc4946..e704bd74a 100644 --- a/src/bsppp/bsppp.cpp +++ b/src/bsppp/bsppp.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include #include @@ -14,10 +15,10 @@ using namespace sourcepp; namespace { template -[[nodiscard]] std::vector readLumpContents(const BSP& bsp, BSPLump lump, void(*callback)(const BSP&, BufferStreamReadOnly&, std::vector&) = [](const BSP&, BufferStreamReadOnly& stream, std::vector& out) { +[[nodiscard]] std::vector parseLumpContents(const BSP& bsp, BSPLump lump, void(*callback)(const BSP&, BufferStreamReadOnly&, std::vector&) = [](const BSP&, BufferStreamReadOnly& stream, std::vector& out) { stream.read(out, stream.size() / sizeof(T)); }) { - auto data = bsp.readLump(lump); + auto data = bsp.getLumpData(lump); if (!data) { return {}; } @@ -30,7 +31,7 @@ template template requires requires(New) { {New::upgrade(Old{})} -> std::same_as; } -void readAndUpgrade(BufferStreamReadOnly& stream, std::vector& out) { +void parseAndUpgrade(BufferStreamReadOnly& stream, std::vector& out) { std::vector old; stream.read(old, stream.size() / sizeof(Old)); for (const auto& elem : old) { @@ -40,324 +41,254 @@ void readAndUpgrade(BufferStreamReadOnly& stream, std::vector& out) { } // namespace -BSP::BSP(std::string path_) +BSP::BSP(std::string path_, bool loadPatchFiles) : path(std::move(path_)) { - FileStream reader{this->path}; - if (!reader) { + if (!this->readHeader()) { this->path.clear(); return; } - if (auto signature = reader.seek_in(0).read(); signature != BSP_SIGNATURE) { - // File is not a BSP - this->path.clear(); - return; - } - this->header.version = reader.read(); + this->stagedVersion = this->header.version; + this->stagedGameLumps = this->parseGameLumps(false); + this->stagedMapRevision = this->header.mapRevision; - // Contagion funny - if (this->header.version == 27) { - reader.skip_in(); - } - - reader >> this->header.lumps; + if (loadPatchFiles) { + const auto fsPath = std::filesystem::path{this->path}; + const auto fsStem = (fsPath.parent_path() / fsPath.stem()).string() + "_l_"; - // If no offsets are larger than 1024 (less than the size of the BSP header, but greater than any lump version), - // it's probably a L4D2 BSP and those are lump versions! - if (this->header.version == 21) { - int i; - for (i = 0; i < BSP_LUMP_COUNT; i++) { - if (this->header.lumps[i].offset > 1024) { + for (int i = 0; ; i++) { + auto patchFilePath = fsStem + std::to_string(i) + ".lmp"; + if (!std::filesystem::exists(patchFilePath)) { break; } - } - this->isL4D2 = i == BSP_LUMP_COUNT; - if (this->isL4D2) { - // Swap fields around - for (i = 0; i < BSP_LUMP_COUNT; i++) { - std::swap(this->header.lumps[i].offset, this->header.lumps[i].version); - std::swap(this->header.lumps[i].offset, this->header.lumps[i].length); - } + this->setLumpFromPatchFile(patchFilePath); } } - - reader >> this->header.mapRevision; - this->stagedHeader = this->header; - this->stagedGameLumps = this->readGameLumps(); } BSP::operator bool() const { return !this->path.empty(); } -BSP BSP::create(std::string path, int32_t version, int32_t mapRevision) { +BSP BSP::create(std::string path, uint32_t version, uint32_t mapRevision) { { FileStream writer{path, FileStream::OPT_TRUNCATE | FileStream::OPT_CREATE_IF_NONEXISTENT}; - writer << BSP_SIGNATURE << BSP::Header{ - .version = version, - .lumps = {}, - .mapRevision = mapRevision, - }; + writer << BSP_SIGNATURE << version; + if (version == 27) { + writer.write(0); + } + writer + .write({}) + .write(mapRevision); } return BSP{std::move(path)}; } -int32_t BSP::getVersion() const { - return this->header.version; +uint32_t BSP::getVersion() const { + return this->stagedVersion; } -void BSP::setVersion(int32_t version) { - this->stagedHeader.version = version; +void BSP::setVersion(uint32_t version) { + this->stagedVersion = version; } -int32_t BSP::getMapRevision() const { - return this->header.mapRevision; +uint32_t BSP::getMapRevision() const { + return this->stagedMapRevision; } -void BSP::setMapRevision(int32_t mapRevision) { - this->stagedHeader.mapRevision = mapRevision; +void BSP::setMapRevision(uint32_t mapRevision) { + this->stagedMapRevision = mapRevision; } bool BSP::hasLump(BSPLump lumpIndex) const { if (this->path.empty()) { return false; } - const auto lump = static_cast>(lumpIndex); - return this->header.lumps[lump].length != 0 && this->header.lumps[lump].offset != 0; + const auto lump = static_cast>(lumpIndex); + return (this->stagedLumps.contains(lump) && this->stagedLumps.at(lump).first.length != 0 && this->stagedLumps.at(lump).first.offset != 0) + || (this->header.lumps[lump].length != 0 && this->header.lumps[lump].offset != 0); } bool BSP::isLumpCompressed(BSPLump lumpIndex) const { - if (this->hasLump(lumpIndex)){ - return this->header.lumps[static_cast>(lumpIndex)].fourCC > 0; - } - return false; + if (this->hasLump(lumpIndex)) { + const auto lump = static_cast>(lumpIndex); + return (this->stagedLumps.contains(lump) && this->stagedLumps.at(lump).first.uncompressedLength > 0) + || (this->header.lumps[lump].uncompressedLength > 0); + } + return false; } -int32_t BSP::getLumpVersion(BSPLump lumpIndex) const { +uint32_t BSP::getLumpVersion(BSPLump lumpIndex) const { if (this->path.empty()) { return 0; } - return this->header.lumps[static_cast>(lumpIndex)].version; -} - -void BSP::setLumpVersion(BSPLump lumpIndex, int32_t version) { - this->stagedHeader.lumps[static_cast>(lumpIndex)].version = version; + const auto lump = static_cast>(lumpIndex); + if (this->stagedLumps.contains(lump)) { + return this->stagedLumps.at(lump).first.version; + } + return this->header.lumps[lump].version; } -std::optional> BSP::readLump(BSPLump lumpIndex, bool readRaw, bool stagedVersion) const { - if (this->path.empty() || !this->hasLump(lumpIndex)) { +std::optional> BSP::getLumpData(BSPLump lumpIndex, bool noDecompression) const { + if (this->path.empty() || !this->hasLump(lumpIndex)) { return std::nullopt; } - auto lumpToRead = static_cast>(lumpIndex); + auto lump = static_cast>(lumpIndex); std::vector lumpBytes; - if (stagedVersion) { - lumpBytes = this->stagedLumps[lumpToRead]; - } - else{ + if (this->stagedLumps.contains(lump)) { + lumpBytes = this->stagedLumps.at(lump).second; + } else { FileStream reader{this->path}; lumpBytes = reader - .seek_in(this->header.lumps[lumpToRead].offset) - .read_bytes(this->header.lumps[lumpToRead].length); + .seek_in(this->header.lumps[lump].offset) + .read_bytes(this->header.lumps[lump].length); } - if ( this->isLumpCompressed(lumpIndex) && !readRaw) { - return decompressLumpData(lumpBytes); - } - else { - // Not Compressed and we dont need to do anything - return lumpBytes; - } - + if (!noDecompression && this->isLumpCompressed(lumpIndex)) { + return decompressLumpData(lumpBytes); + } + return lumpBytes; } -bool BSP::stageLump(BSPLump lumpIndex, const std::span data, uint8_t compressLevel) { - +bool BSP::setLump(BSPLump lumpIndex, uint32_t version, std::span data, uint8_t compressLevel) { if (this->path.empty() || lumpIndex == BSPLump::UNKNOWN) { - return false; - } - - auto lumpToStage = static_cast>(lumpIndex); - - if (compressLevel > 0) { - auto newLump = compressLumpData(data, compressLevel); - if (!newLump.has_value()) - return false; - - this->stagedHeader.lumps[lumpToStage].fourCC = data.size(); - this->stagedLumps[lumpToStage].clear(); - this->stagedLumps[lumpToStage].insert(stagedLumps[lumpToStage].begin(), newLump.value().begin(), newLump.value().end()); - } - else { - this->stagedHeader.lumps[lumpToStage].fourCC = 0; - this->stagedLumps[lumpToStage].clear(); - this->stagedLumps[lumpToStage].insert(stagedLumps[lumpToStage].begin(), data.begin(), data.end()); - } - - return true; -} - -bool BSP::stageLump(std::span entities, uint8_t compressLevel) { - if (this->path.empty()) { - return false; - } - - std::vector byteVec; - for (auto e : entities) - { - byteVec.push_back((std::byte)'{'); - for (auto it = e.begin(); it != e.end(); it++) { - byteVec.push_back((std::byte)'\"'); - - for (char c : it->first){ - byteVec.push_back((std::byte)c); - } - byteVec.push_back((std::byte)'\"'); - byteVec.push_back((std::byte)' '); - byteVec.push_back((std::byte)'\"'); - for (char c : it->second){ - byteVec.push_back((std::byte)c); - } - byteVec.push_back((std::byte)'\"'); - } - byteVec.push_back((std::byte)'}'); - } - return this->stageLump(BSPLump::ENTITIES, byteVec, compressLevel); -} - -void BSP::writeChangesToDisk() { - uint32_t offset = sizeof(BSP_SIGNATURE) + sizeof(BSP::Header); - this->header = this->stagedHeader; - - for ( int32_t i = 0; i < BSP_LUMP_COUNT; i++){ // Fill empty space with lumps we're keeping - if (i == static_cast(BSPLump::GAME_LUMP) - && this->stagedGameLumps.size() > 0) {// Need special handling since its split up - - uint32_t gameLumpOffset = offset; - if (this->stagedGameLumps[0].data.size() != this->stagedGameLumps[0].length) { // - // length member is decompressed size, need to add an empty item with an offset so the compressed size is known later - BSPGameLump dummy{0,0,0,0,0}; - this->stagedGameLumps.push_back(dummy); - } - - auto &lump = this->stagedLumps[i]; - - lump.clear(); - BufferStream stream(lump); - stream.write(this->stagedGameLumps.size()); - gameLumpOffset += sizeof(int32_t); - gameLumpOffset += (sizeof(BSPGameLump) - sizeof(BSPGameLump::data)) * this->stagedGameLumps.size(); - //gameLumpOffset += 128 * this->stagedGameLumps.size(); - - for (auto gameLump : this->stagedGameLumps) { - if (gameLump.id == 0) { - gameLump.fileoffset = 0; - } - else { - gameLump.fileoffset = gameLumpOffset; - gameLumpOffset += gameLump.data.size(); - } - stream.write(gameLump.id) - .write(gameLump.flags) - .write(gameLump.version) - .write(gameLump.fileoffset) - .write(gameLump.length); - } - for (auto gameLump : this->stagedGameLumps) - { - if (gameLump.id == 0) - continue; - stream.write(gameLump.data); - } - this->header.lumps[i].offset = offset; - this->header.lumps[i].length = lump.size(); - offset += lump.size(); - - } - else { - auto &lump = this->stagedLumps[i]; - if (lump.size() == 0){ - if (this->hasLump(static_cast(i))) - lump = this->readLump(static_cast(i), true).value(); - } - - if (lump.size() == 0) - this->header.lumps[i].offset = 0; - else{ - this->header.lumps[i].offset = offset; - offset += lump.size(); - } - this->header.lumps[i].length = lump.size(); - } - } - - this->writeHeader(); - FileStream writer{ this->path, FileStream::OPT_READ | FileStream::OPT_WRITE }; - - for ( int i = 0; i < BSP_LUMP_COUNT; i++){ - if (this->stagedLumps[i].size()) - writer.seek_out(this->header.lumps[i].offset) - .write(this->stagedLumps[i]); - } -} - -void BSP::flushChanges(BSPLump lumpIndex) -{ - if (lumpIndex == BSPLump::UNKNOWN) { - for (int i = 0; i < static_cast>(BSPLump::COUNT); i++) { - this->flushChanges(static_cast(i)); - } - return; + return false; + } + if (lumpIndex == BSPLump::GAME_LUMP) { + // Game lump needs to be modified with other methods + return false; } + if (lumpIndex == BSPLump::PAKFILE) { + // Paklump should use zip compression + compressLevel = 0; + } - if (lumpIndex == BSPLump::GAME_LUMP) { - this->stagedGameLumps = this->readGameLumps(); + const auto lump = static_cast>(lumpIndex); + if (compressLevel > 0) { + auto compressedData = compressLumpData(data, compressLevel); + if (!compressedData) { + return false; + } + this->stagedLumps[lump] = std::make_pair>({ + .version = version, .uncompressedLength = static_cast(data.size()), + }, {compressedData->begin(), compressedData->end()}); + } else { + this->stagedLumps[lump] = std::make_pair>({ + .version = version, .uncompressedLength = 0, + }, {data.begin(), data.end()}); } - auto lumpToFlush = static_cast>(lumpIndex); - this->stagedLumps[lumpToFlush].clear(); - this->stagedHeader.lumps[lumpToFlush] = this->header.lumps[lumpToFlush]; + return true; } -bool BSP::stageLumpPatchFile(const std::string& lumpFilePath) { - if (this->path.empty()) { - return false; +bool BSP::isGameLumpCompressed(BSPGameLump::Signature signature) const { + for (const auto& gameLump : this->stagedGameLumps) { + if (gameLump.signature == signature) { + return gameLump.isCompressed; + } } + return false; +} - FileStream reader{lumpFilePath}; - if (!reader) { - return false; +uint16_t BSP::getGameLumpVersion(BSPGameLump::Signature signature) { + for (const auto& gameLump : this->stagedGameLumps) { + if (gameLump.signature == signature) { + return gameLump.version; + } } + return 0; +} - const auto offset = reader.read(); - const auto index = reader.read(); - const auto version = reader.read(); - const auto length = reader.read(); - if (index < 0 || index > BSP_LUMP_COUNT || offset <= 0 || length <= 0) { - return false; +std::optional> BSP::getGameLumpData(BSPGameLump::Signature signature) const { + for (const auto& gameLump : this->stagedGameLumps) { + if (gameLump.signature == signature) { + if (gameLump.isCompressed) { + return decompressLumpData(gameLump.data); + } + return gameLump.data; + } } + return std::nullopt; +} + +bool BSP::setGameLump(BSPGameLump::Signature signature, uint16_t version, std::span data, uint8_t compressLevel) { + BSPGameLump gameLump{ + .signature = signature, + .isCompressed = compressLevel > 0, + .version = version, + .offset = 0, + .uncompressedLength = static_cast(data.size()), + }; - this->header.lumps[index].version = version; - this->stageLump(static_cast(index), reader.seek_in(offset).read_bytes(length)); + if (compressLevel) { + const auto compressedData = compressLumpData(data, compressLevel); + if (!compressedData) { + return false; + } + gameLump.data = *compressedData; + } else { + gameLump.data = {data.begin(), data.end()}; + } + + this->stagedGameLumps.push_back(gameLump); return true; } +void BSP::resetLump(BSPLump lumpIndex) { + if (this->path.empty()) { + return; + } + if (lumpIndex == BSPLump::UNKNOWN) { + this->reset(); + return; + } + if (lumpIndex == BSPLump::GAME_LUMP) { + this->stagedGameLumps = this->parseGameLumps(false); + return; + } + const auto lump = static_cast>(lumpIndex); + if (this->stagedLumps.contains(lump)) { + this->stagedLumps.erase(lump); + } +} + +void BSP::reset() { + if (this->path.empty()) { + return; + } + this->stagedVersion = this->header.version; + this->stagedLumps.clear(); + this->stagedGameLumps = this->parseGameLumps(false); + this->stagedMapRevision = this->header.mapRevision; +} + void BSP::createLumpPatchFile(BSPLump lumpIndex) const { - auto lumpData = this->readLump(lumpIndex); + if (this->path.empty()) { + return; + } + + auto lumpData = this->getLumpData(lumpIndex); if (!lumpData) { return; } - const auto& [lumpOffset, lumpLength, lumpVersion, lumpFourCC] = this->header.lumps.at(static_cast>(lumpIndex)); + const auto& [ + lumpOffset, + lumpLength, + lumpVersion, + lumpUncompressedLength + ] = this->header.lumps.at(static_cast>(lumpIndex)); const auto fsPath = std::filesystem::path{this->path}; const auto fsStem = (fsPath.parent_path() / fsPath.stem()).string() + "_l_"; - int nonexistentNumber; - for (nonexistentNumber = 0; true; nonexistentNumber++) { + int nonexistentNumber = 0; + while (1) { if (!std::filesystem::exists(fsStem + std::to_string(nonexistentNumber) + ".lmp")) { break; } + nonexistentNumber++; } FileStream writer{fsStem + std::to_string(nonexistentNumber) + ".lmp", FileStream::OPT_TRUNCATE | FileStream::OPT_CREATE_IF_NONEXISTENT}; @@ -371,333 +302,413 @@ void BSP::createLumpPatchFile(BSPLump lumpIndex) const { .write(*lumpData); } -bool BSP::stageGameLump(char id[4], uint16_t flags, uint16_t version, std::span data, uint8_t compressLevel) { - return this->stageGameLump(id[0] | (id[1] << 8) | (id[2] << 16) | (id[3] << 24) - , flags, version, data, compressLevel); -} - -bool BSP::stageGameLump(int32_t id, uint16_t flags, uint16_t version, std::span data, uint8_t compressLevel) { - BSPGameLump gameLump; - gameLump.id = id; - gameLump.flags = flags; - gameLump.version = version; - gameLump.length = data.size(); - - if (compressLevel){ - auto compressed = compressLumpData(data, compressLevel); - if (!compressed.has_value()) - return false; - gameLump.data = compressed.value(); - } else { - gameLump.data.clear(); - gameLump.data.insert(gameLump.data.begin(), data.begin(), data.end()); +bool BSP::setLumpFromPatchFile(const std::string& lumpFilePath) { + if (this->path.empty()) { + return false; } - this->stagedGameLumps.push_back(gameLump); - return true; -} - -void BSP::writeHeader() const { - FileStream writer{this->path, FileStream::OPT_READ | FileStream::OPT_WRITE}; - writer.seek_out(0) << BSP_SIGNATURE << this->header.version; - // Contagion funny - if (this->header.version == 27) { - writer.write(0); + FileStream reader{lumpFilePath}; + if (!reader) { + return false; } - if (!this->isL4D2) { - writer << this->header.lumps; - } else { - for (int i = 0; i < static_cast>(BSPLump::COUNT); i++) { - writer << this->header.lumps[i].version << this->header.lumps[i].offset << this->header.lumps[i].length << this->header.lumps[i].fourCC; - } + const auto offset = reader.read(); + const auto index = reader.read(); + const auto version = reader.read(); + const auto length = reader.read(); + if (index > BSP_LUMP_COUNT || offset == 0 || length == 0) { + return false; } - writer << this->header.mapRevision; + this->setLump(static_cast(index), version, reader.seek_in(offset).read_bytes(length)); + return true; } -std::vector BSP::parseEntities() const { - auto lumpData = this->readLump(BSPLump::ENTITIES); +bool BSP::bake(std::string_view outputPath) { + if (this->path.empty()) { + return false; + } - std::vector ents; + std::vector out; + BufferStream stream{out}; - if (!lumpData.has_value()) - { - return ents; + stream << BSP_SIGNATURE << this->stagedVersion; + if (this->stagedVersion == 27) { + // Contagion funny + stream.write(0); } - auto lumpDataValue = lumpData.value(); + const auto lumpsHeaderOffset = stream.tell(); + stream.write({}); - BSPEntity e; - std::string key, value; - bool quoted = false; - bool keying = true; + stream << this->stagedMapRevision; - for (uint32_t i = 0; i hasLump(static_cast(i))) { continue; } - if ( c == '\"') { - quoted = !quoted; - if (!quoted && !keying) { - e[key] = value; - key.erase(); - value.erase(); + if (static_cast(i) == BSPLump::GAME_LUMP && !this->stagedGameLumps.empty()) { + const auto gameLumpOffset = stream.tell(); + + bool oneOrMoreGameLumpCompressed = false; + for (auto gameLump : this->stagedGameLumps) { + if (gameLump.isCompressed) { + oneOrMoreGameLumpCompressed = true; + break; + } + } + auto gameLumpCurrentOffset = sizeof(int32_t) + ((sizeof(BSPGameLump) - sizeof(BSPGameLump::data)) * (this->stagedGameLumps.size() + oneOrMoreGameLumpCompressed)); + stream.write(this->stagedGameLumps.size() + oneOrMoreGameLumpCompressed); + + for (const auto& gameLump : this->stagedGameLumps) { + if (gameLump.signature == 0) { + break; + } + stream + .write(gameLump.signature) + .write(gameLump.isCompressed) + .write(gameLump.version) + .write(gameLumpCurrentOffset) + .write(gameLump.uncompressedLength); + gameLumpCurrentOffset += gameLump.data.size(); + } + if (oneOrMoreGameLumpCompressed) { + stream + .write(0) + .write(0) + .write(0) + .write(0) + .write(0); + } + + for (const auto& gameLump : this->stagedGameLumps) { + if (gameLump.signature == 0) { + break; + } + stream.write(gameLump.data); } - if (!quoted) { - keying = !keying; + const auto curPos = stream.tell(); + stream.seek_u(lumpsHeaderOffset + (i * sizeof(Lump))); + if (!this->isL4D2) { + stream + .write(gameLumpOffset) + .write(curPos - gameLumpOffset) + .write(0); + } else { + stream + .write(0) + .write(gameLumpOffset) + .write(curPos - gameLumpOffset); } + stream + .write(0) + .seek_u(curPos); continue; } - if ( c == '}') { - ents.push_back(e); - e.clear(); + + if (this->stagedLumps.contains(i)) { + const auto& lumpPair = this->stagedLumps.at(i); + const auto curPos = stream.tell(); + stream.seek_u(lumpsHeaderOffset + (i * sizeof(Lump))); + if (!this->isL4D2) { + stream + .write(curPos) + .write(lumpPair.second.size()) + .write(lumpPair.first.version); + } else { + stream + .write(lumpPair.first.version) + .write(curPos) + .write(lumpPair.second.size()); + } + stream + .write(lumpPair.first.uncompressedLength) + .seek_u(curPos) + .write(lumpPair.second); continue; } - if (!quoted) - continue; - if ( keying ) { - key += c; + const auto data = this->getLumpData(static_cast(i), true); + if (data) { + const auto curPos = stream.tell(); + stream.seek_u(lumpsHeaderOffset + (i * sizeof(Lump))); + if (!this->isL4D2) { + stream + .write(curPos) + .write(this->header.lumps[i].length) + .write(this->header.lumps[i].version); + } else { + stream + .write(this->header.lumps[i].version) + .write(curPos) + .write(this->header.lumps[i].length); + } + stream + .write(this->header.lumps[i].uncompressedLength) + .seek_u(curPos) + .write(*data); } - else { - value += c; + } + + out.resize(stream.tell()); + if (!fs::writeFileBuffer(outputPath.empty() ? this->path : std::string{outputPath}, out)) { + return false; + } + + this->path = outputPath; + this->stagedLumps.clear(); + return this->readHeader(); +} + +bool BSP::readHeader() { + FileStream reader{this->path}; + if (!reader) { + return false; + } + reader.seek_in(0); + + if (reader.read() != BSP_SIGNATURE) { + // File is not a BSP + return false; + } + this->header.version = reader.read(); + + // Contagion funny + if (this->header.version == 27) { + reader.skip_in(); + } + + reader >> this->header.lumps; + + // If no offsets are larger than 1024 (less than the size of the BSP header, but greater + // than any lump version), it's probably a L4D2 BSP and those are lump versions! + if (this->header.version == 21) { + int i; + for (i = 0; i < BSP_LUMP_COUNT; i++) { + if (this->header.lumps[i].offset > 1024) { + break; + } + } + this->isL4D2 = i == BSP_LUMP_COUNT; + if (this->isL4D2) { + // Swap fields around + for (i = 0; i < BSP_LUMP_COUNT; i++) { + std::swap(this->header.lumps[i].offset, this->header.lumps[i].version); + std::swap(this->header.lumps[i].offset, this->header.lumps[i].length); + } } } - if ( quoted || !keying ) // somethings messed up - ents.clear(); + reader >> this->header.mapRevision; + return true; +} - return ents; +std::vector BSP::parsePlanes() const { + return ::parseLumpContents(*this, BSPLump::PLANES); } -std::vector BSP::readPlanes() const { - return ::readLumpContents(*this, BSPLump::PLANES); +std::vector BSP::parseTextureData() const { + return ::parseLumpContents(*this, BSPLump::TEXDATA); } -std::vector BSP::readTextureData() const { - return ::readLumpContents(*this, BSPLump::TEXDATA); +std::vector BSP::parseVertices() const { + return ::parseLumpContents(*this, BSPLump::VERTEXES); } -std::vector BSP::readVertices() const { - return ::readLumpContents(*this, BSPLump::VERTEXES); +std::vector BSP::parseNodes() const { + return ::parseLumpContents(*this, BSPLump::NODES, [](const BSP& bsp, BufferStreamReadOnly& stream, std::vector& out) { + if (const auto lumpVersion = bsp.getLumpVersion(BSPLump::NODES); lumpVersion == 1) { + stream.read(out, stream.size() / sizeof(BSPNode_v1)); + } else if (lumpVersion == 0) { + ::parseAndUpgrade(stream, out); + } + }); } -std::vector BSP::readTextureInfo() const { - return ::readLumpContents(*this, BSPLump::TEXINFO); +std::vector BSP::parseTextureInfo() const { + return ::parseLumpContents(*this, BSPLump::TEXINFO); } -std::vector BSP::readFaces() const { - return ::readLumpContents(*this, BSPLump::FACES, [](const BSP& bsp, BufferStreamReadOnly& stream, std::vector& out) { - if (bsp.getLumpVersion(BSPLump::FACES) > 1) { +std::vector BSP::parseFaces() const { + return ::parseLumpContents(*this, BSPLump::FACES, [](const BSP& bsp, BufferStreamReadOnly& stream, std::vector& out) { + if (const auto lumpVersion = bsp.getLumpVersion(BSPLump::FACES); lumpVersion == 2) { stream.read(out, stream.size() / sizeof(BSPFace_v2)); - } else { - ::readAndUpgrade(stream, out); + } else if (lumpVersion == 1) { + ::parseAndUpgrade(stream, out); } }); } -std::vector BSP::readEdges() const { - return ::readLumpContents(*this, BSPLump::EDGES, [](const BSP& bsp, BufferStreamReadOnly& stream, std::vector& out) { - if (bsp.getLumpVersion(BSPLump::EDGES) > 0) { +std::vector BSP::parseEdges() const { + return ::parseLumpContents(*this, BSPLump::EDGES, [](const BSP& bsp, BufferStreamReadOnly& stream, std::vector& out) { + if (const auto lumpVersion = bsp.getLumpVersion(BSPLump::EDGES); lumpVersion == 1) { stream.read(out, stream.size() / sizeof(BSPEdge_v1)); - } else { - ::readAndUpgrade(stream, out); + } else if (lumpVersion == 0) { + ::parseAndUpgrade(stream, out); } }); } -std::vector BSP::readSurfEdges() const { - return ::readLumpContents(*this, BSPLump::SURFEDGES); +std::vector BSP::parseSurfEdges() const { + return ::parseLumpContents(*this, BSPLump::SURFEDGES); } -std::vector BSP::readBrushModels() const { - return ::readLumpContents(*this, BSPLump::MODELS); +std::vector BSP::parseBrushModels() const { + return ::parseLumpContents(*this, BSPLump::MODELS); } -std::vector BSP::readOriginalFaces() const { - return ::readLumpContents(*this, BSPLump::ORIGINALFACES, [](const BSP& bsp, BufferStreamReadOnly& stream, std::vector& out) { +std::vector BSP::parseOriginalFaces() const { + return ::parseLumpContents(*this, BSPLump::ORIGINALFACES, [](const BSP& bsp, BufferStreamReadOnly& stream, std::vector& out) { // ORIGINALFACES lump version is always 0? - if (bsp.getLumpVersion(BSPLump::FACES) > 1) { + if (const auto lumpVersion = bsp.getLumpVersion(BSPLump::FACES); lumpVersion == 2) { stream.read(out, stream.size() / sizeof(BSPFace_v2)); - } else { - ::readAndUpgrade(stream, out); + } else if (lumpVersion == 1) { + ::parseAndUpgrade(stream, out); } }); } -std::vector BSP::readGameLumps() const { +std::vector BSP::parseGameLumps(bool decompress) const { std::vector lumps; - auto lumpsAggregate = this->readLump(BSPLump::GAME_LUMP, true); - if (!lumpsAggregate.has_value()) + auto gameLumpData = this->getLumpData(BSPLump::GAME_LUMP); + if (!gameLumpData) { return lumps; + } + BufferStreamReadOnly stream{*gameLumpData}; - BufferStreamReadOnly stream{lumpsAggregate.value()}; - - uint32_t numGameLumps = stream.read(); - for (int i = 0; i < numGameLumps; i++) - { - BSPGameLump lump; + lumps.resize(stream.read()); + for (auto& lump : lumps) { stream - .read(lump.id) - .read(lump.flags) + .read(lump.signature) + .read(lump.isCompressed) .read(lump.version) - .read(lump.fileoffset) - .read(lump.length); - // if (lump.id != 0) - lumps.push_back(lump); + .read(lump.offset) + .read(lump.uncompressedLength); } - bool isCompressed = lumps.back().id == 0; // Game Lumps are compressed individually, - //when they are there is an empty entry with only the offset to calculate compressed size - for (int i = 0; i < numGameLumps; i++) { - if (lumps[i].id == 0) + // When one or more game lumps are compressed, the last entry + // is empty except the offset field to calculate compressed + // size. Game lumps are compressed individually + for (uint32_t i = 0; i < lumps.size(); i++) { + if (lumps[i].signature == 0) { break; - if (!isCompressed) { - lumps[i].data = stream.read_bytes(lumps[i].length); + } + if (!lumps[i].isCompressed) { + lumps[i].data = stream.read_bytes(lumps[i].uncompressedLength); } else { - //printf("%i: ", i); - int32_t nextOffset = lumps[i+1].fileoffset; + auto nextOffset = lumps[i + 1].offset; if (nextOffset == 0) { - auto id = static_cast>(BSPLump::GAME_LUMP); + const auto id = static_cast>(BSPLump::GAME_LUMP); nextOffset = this->header.lumps[id].offset + this->header.lumps[id].length; } - int32_t length = nextOffset - lumps[i].fileoffset; - //printf("%i\n", length); - auto data = stream.read_bytes(length); - auto decompressed = decompressLumpData(data); - - if (!decompressed.has_value()) - lumps[i].data = data; - else - lumps[i].data = decompressed.value(); + if (!decompress) { + lumps[i].data = stream.read_bytes(nextOffset - lumps[i].offset); + } else { + auto uncompressedData = decompressLumpData(stream.read_bytes(nextOffset - lumps[i].offset)); + if (uncompressedData) { + lumps[i].data = *uncompressedData; + } + } } } - if (lumps.back().id == 0) - lumps.pop_back(); + if (lumps.back().signature == 0) { + lumps.pop_back(); + } return lumps; - } -std::optional> BSP::compressLumpData(const std::span data, uint8_t compressLevel){ - uint32_t bufsize = data.size_bytes() + sizeof(lzma_header_alone); - std::byte *compressedData = (std::byte*)malloc(bufsize); - lzma_stream stream = LZMA_STREAM_INIT; +std::optional> BSP::compressLumpData(std::span data, uint8_t compressLevel) { + // Preallocate extra 4 bytes for Valve LZMA header signature + std::vector compressedData(sizeof(uint32_t)); + std::array compressedChunk{}; - stream.next_in = (uint8_t*)data.data(); - stream.avail_in = data.size_bytes(); // + 32; - stream.next_out = (uint8_t*)compressedData; - stream.avail_out = bufsize; + lzma_stream stream{ + .next_in = reinterpret_cast(data.data()), + .avail_in = data.size(), + .next_out = reinterpret_cast(compressedChunk.data()), + .avail_out = compressedChunk.size(), + }; - lzma_options_lzma options; - if ( compressLevel > 9 )// Out of allowed range - compressLevel = 6; - lzma_lzma_preset(&options, compressLevel); - lzma_ret initError = lzma_alone_encoder(&stream, &options); + lzma_options_lzma options{}; + lzma_lzma_preset(&options, std::clamp(compressLevel, 0, 9)); - if (initError) { + if (lzma_alone_encoder(&stream, &options) != LZMA_OK) { lzma_end(&stream); - //printf("init %i", initError); - free(compressedData); return std::nullopt; } - lzma_ret compressError; - while (true){//(compressError == LZMA_OK) { - compressError = lzma_code(&stream, LZMA_RUN); - // if (stream.total_in >= data.size()) { - // break; - // } - if (compressError != LZMA_OK) - { - break; + lzma_ret ret; + do { + stream.next_out = reinterpret_cast(compressedChunk.data()); + stream.avail_out = compressedChunk.size(); + + ret = lzma_code(&stream, LZMA_RUN); + compressedData.insert(compressedData.end(), compressedChunk.begin(), compressedChunk.begin() + compressedChunk.size() - stream.avail_out); + } while (ret == LZMA_OK); + + if (auto code = lzma_code(&stream, LZMA_FINISH); code != LZMA_OK && code != LZMA_STREAM_END) { + lzma_end(&stream); + return std::nullopt; + } + lzma_end(&stream); + + { + // Switch out normal header with Valve one + BufferStream compressedStream{compressedData}; + compressedStream << LZMA_VALVE_SIGNATURE; + const auto properties = compressedStream.read>(); + compressedStream + .seek_u(sizeof(uint32_t)) + .write(data.size()) + .write(compressedData.size() - (sizeof(uint32_t) * 3) + (sizeof(uint8_t) * 5)) + .write(properties); + } + return compressedData; +} + +std::optional> BSP::decompressLumpData(std::span data) { + std::vector compressedData{data.begin(), data.end()}; + std::vector uncompressedData; + { + // Switch out Valve header with normal one + BufferStreamReadOnly in{data.data(), data.size()}; + if (in.read() != LZMA_VALVE_SIGNATURE) { + return std::nullopt; } + const auto uncompressedLength = in.read(); + in.skip(); + const auto properties = in.read>(); + BufferStream out{compressedData}; + out + .write(properties) + .write(uncompressedLength); + uncompressedData.resize(uncompressedLength); + } + + lzma_stream stream{ + .next_in = reinterpret_cast(compressedData.data()), + .avail_in = compressedData.size(), + .next_out = reinterpret_cast(uncompressedData.data()), + .avail_out = uncompressedData.size(), + }; + + if (lzma_alone_decoder(&stream, UINT64_MAX) != LZMA_OK) { + lzma_end(&stream); + return std::nullopt; } - // printf("%i\n", compressError); - //if (compressError == LZMA_STREAM_END) - //{ - compressError = lzma_code(&stream, LZMA_RUN); - compressError = lzma_code(&stream, LZMA_FINISH); + if (auto ret = lzma_code(&stream, LZMA_RUN); ret != LZMA_OK && ret != LZMA_STREAM_END) { lzma_end(&stream); - //} - - // if (compressError != LZMA_OK) { - // //printf("compress %i\n", compressError); - // lzma_end(&stream); - // return false; - // } - - std::vector compressedLump; - //newLump.insert(newLump.begin(), compressedData, compressedData + stream.total_out); - - // Transplant header with Valve's - lzma_header_bsplump headerBSP; - lzma_header_alone *headerAlone = (lzma_header_alone*)compressedData; - - headerBSP.id = BSP_LZMA_ID; - headerBSP.actualSize = stream.total_in; - headerBSP.lzmaSize = stream.total_out - sizeof(lzma_header_alone); - - //printf("actual: %i\nbsp: %i\n", headerAlone->actualSize, headerBSP.actualSize); - - std::copy(headerAlone->properties, headerAlone->properties + sizeof(headerAlone->properties), headerBSP.properties); - - //newLump.erase(newLump.begin(), newLump.begin() + sizeof(lzma_header_alone)); - //newLump.insert(newLump.begin(), (std::byte*)&headerBSP, ((std::byte*)&headerBSP) + sizeof(lzma_header_bsplump)); - compressedLump.insert(compressedLump.begin(), (std::byte*)&headerBSP, (std::byte*)&headerBSP + sizeof(headerBSP)); - compressedLump.insert(compressedLump.end(), compressedData + sizeof(lzma_header_alone), compressedData + stream.total_out); - free(compressedData); - return compressedLump; -} - -std::optional> BSP::decompressLumpData(const std::span data) { - // Transplant Valve header with a normal one - lzma_header_bsplump *headerBSP = (lzma_header_bsplump*)(data.data()); - lzma_header_alone headerAlone; - headerAlone.actualSize = headerBSP->actualSize; - - std::copy(headerBSP->properties, headerBSP->properties + sizeof(headerBSP->properties), headerAlone.properties); - int32_t uncompressedSize = headerAlone.actualSize; - //data.erase(data.begin(), data.begin() + sizeof(lzma_header_bsplump)); - //data.insert(data.begin(), (std::byte*)&headerAlone, ((std::byte*)&headerAlone) + sizeof(lzma_header_alone)); - std::span fixedHeaderData = data.subspan(sizeof(lzma_header_bsplump) - sizeof(lzma_header_alone), std::dynamic_extent); - *((lzma_header_alone*)fixedHeaderData.data()) = headerAlone; - - std::byte *decodedLump = (std::byte*)malloc(uncompressedSize); - - uint64_t memlimit = 1074000000 ; // 1 GB should be more than enough - lzma_stream stream = LZMA_STREAM_INIT; - stream.next_in = (uint8_t*)fixedHeaderData.data(); - stream.avail_in = fixedHeaderData.size(); - stream.next_out = (uint8_t*)decodedLump; - stream.avail_out = uncompressedSize; - lzma_ret initError = lzma_alone_decoder(&stream, memlimit); - - if (initError) { - //printf("init error %i\n", initError); - lzma_end(&stream); - free(decodedLump); - return std::nullopt; - } - - lzma_ret decompressError; - decompressError = lzma_code(&stream, LZMA_RUN); - decompressError = lzma_code(&stream, LZMA_FINISH); - lzma_end(&stream); - - if (decompressError != LZMA_STREAM_END && decompressError != LZMA_OK) { - return std::nullopt; - } - - std::vector decodedLumpVec; - decodedLumpVec.insert(decodedLumpVec.begin(), decodedLump, decodedLump + uncompressedSize); - free(decodedLump); - return decodedLumpVec; + return std::nullopt; + } + if (auto ret = lzma_code(&stream, LZMA_FINISH); ret != LZMA_OK && ret != LZMA_STREAM_END) { + lzma_end(&stream); + return std::nullopt; + } + lzma_end(&stream); + + return uncompressedData; } From 82e1f7e424bb9e0b2767ab227b7412695b256b31 Mon Sep 17 00:00:00 2001 From: craftablescience Date: Mon, 6 Jan 2025 01:41:42 -0500 Subject: [PATCH 3/6] bsppp: move BSP class code out to BSP.h/cpp --- include/bsppp/BSP.h | 249 +++++++++++++++++++++++++++++++ include/bsppp/bsppp.h | 249 +------------------------------ src/bsppp/{bsppp.cpp => BSP.cpp} | 2 +- src/bsppp/PakLump.cpp | 2 +- src/bsppp/_bsppp.cmake | 3 +- 5 files changed, 258 insertions(+), 247 deletions(-) create mode 100644 include/bsppp/BSP.h rename src/bsppp/{bsppp.cpp => BSP.cpp} (99%) diff --git a/include/bsppp/BSP.h b/include/bsppp/BSP.h new file mode 100644 index 000000000..7df7bb59c --- /dev/null +++ b/include/bsppp/BSP.h @@ -0,0 +1,249 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "LumpData.h" +#include "PakLump.h" + +namespace bsppp { + +constexpr auto BSP_SIGNATURE = sourcepp::parser::binary::makeFourCC("VBSP"); +constexpr auto LZMA_VALVE_SIGNATURE = sourcepp::parser::binary::makeFourCC("LZMA"); + +enum class BSPLump : int32_t { + UNKNOWN = -1, + ENTITIES = 0, + PLANES, + TEXDATA, + VERTEXES, + VISIBILITY, + NODES, + TEXINFO, + FACES, + LIGHTING, + OCCLUSION, + LEAFS, + FACEIDS, + EDGES, + SURFEDGES, + MODELS, + WORLDLIGHTS, + LEAFFACES, + LEAFBRUSHES, + BRUSHES, + BRUSHSIDES, + AREAS, + AREAPORTALS, + S2004_PORTALS, + UNUSED0 = S2004_PORTALS, + SL4D2_PROPCOLLISION = S2004_PORTALS, + S2004_CLUSTERS, + UNUSED1 = S2004_CLUSTERS, + SL4D2_PROPHULLS = S2004_CLUSTERS, + S2004_PORTALVERTS, + UNUSED2 = S2004_PORTALVERTS, + SL4D2_PROPHULLVERTS = S2004_PORTALVERTS, + S2004_CLUSTERPORTALS, + UNUSED3 = S2004_CLUSTERPORTALS, + SL4D2_PROPTRIS = S2004_CLUSTERPORTALS, + DISPINFO, + ORIGINALFACES, + PHYSDISP, + PHYSCOLLIDE, + VERTNORMALS, + VERTNORMALINDICES, + S2004_DISP_LIGHTMAP_ALPHAS, + UNUSED4 = S2004_DISP_LIGHTMAP_ALPHAS, + DISP_VERTS, + DISP_LIGHTMAP_SAMPLE_POSITIONS, + GAME_LUMP, + LEAFWATERDATA, + PRIMITIVES, + PRIMVERTS, + PRIMINDICES, + PAKFILE, + CLIPPORTALVERTS, + CUBEMAPS, + TEXDATA_STRING_DATA, + TEXDATA_STRING_TABLE, + OVERLAYS, + LEAFMINDISTTOWATER, + FACE_MACRO_TEXTURE_INFO, + DISP_TRIS, + S2004_PHYSCOLLIDESURFACE, + UNUSED5 = S2004_PHYSCOLLIDESURFACE, + SL4D2_PROP_BLOB = S2004_PHYSCOLLIDESURFACE, + WATEROVERLAYS, + S2006_XBOX_LIGHTMAPPAGES, + LEAF_AMBIENT_INDEX_HDR = S2006_XBOX_LIGHTMAPPAGES, + S2006_XBOX_LIGHTMAPPAGEINFOS, + LEAF_AMBIENT_INDEX = S2006_XBOX_LIGHTMAPPAGEINFOS, + LIGHTING_HDR, + WORLDLIGHTS_HDR, + LEAF_AMBIENT_LIGHTING_HDR, + LEAF_AMBIENT_LIGHTING, + XBOX_XZIPPAKFILE, + FACES_HDR, + MAP_FLAGS, + OVERLAY_FADES, + L4D_OVERLAY_SYSTEM_LEVELS, + UNUSED6 = L4D_OVERLAY_SYSTEM_LEVELS, + L4D2_PHYSLEVEL, + UNUSED7 = L4D2_PHYSLEVEL, + ASW_DISP_MULTIBLEND, + UNUSED8 = ASW_DISP_MULTIBLEND, + + COUNT, +}; +constexpr int32_t BSP_LUMP_COUNT = 64; +static_assert(static_cast>(BSPLump::COUNT) == BSP_LUMP_COUNT, "Incorrect lump count!"); + +class BSP { + struct Lump { + /// Byte offset into file + uint32_t offset; + /// Length of lump data + uint32_t length; + /// Lump format version + uint32_t version; + /// Uncompressed length if lump is compressed, else 0 + uint32_t uncompressedLength; + }; + + struct Header { + /// Version of the BSP file + uint32_t version; + /// Lump metadata + std::array lumps; + /// Map version number + uint32_t mapRevision; + }; + +public: + explicit BSP(std::string path_, bool loadPatchFiles = true); + + explicit operator bool() const; + + static BSP create(std::string path, uint32_t version = 21, uint32_t mapRevision = 0); + + [[nodiscard]] uint32_t getVersion() const; + + void setVersion(uint32_t version); + + [[nodiscard]] uint32_t getMapRevision() const; + + void setMapRevision(uint32_t mapRevision); + + [[nodiscard]] bool hasLump(BSPLump lumpIndex) const; + + [[nodiscard]] bool isLumpCompressed(BSPLump lumpIndex) const; + + [[nodiscard]] uint32_t getLumpVersion(BSPLump lumpIndex) const; + + [[nodiscard]] std::optional> getLumpData(BSPLump lumpIndex, bool noDecompression = false) const; + + template + [[nodiscard]] auto getLumpData() const { + if constexpr (Lump == BSPLump::PLANES) { + return this->parsePlanes(); + } else if constexpr (Lump == BSPLump::TEXDATA) { + return this->parseTextureData(); + } else if constexpr (Lump == BSPLump::VERTEXES) { + return this->parseVertices(); + } else if constexpr (Lump == BSPLump::NODES) { + return this->parseNodes(); + } else if constexpr (Lump == BSPLump::TEXINFO) { + return this->parseTextureInfo(); + } else if constexpr (Lump == BSPLump::FACES) { + return this->parseFaces(); + } else if constexpr (Lump == BSPLump::EDGES) { + return this->parseEdges(); + } else if constexpr (Lump == BSPLump::SURFEDGES) { + return this->parseSurfEdges(); + } else if constexpr (Lump == BSPLump::MODELS) { + return this->parseBrushModels(); + } else if constexpr (Lump == BSPLump::ORIGINALFACES) { + return this->parseOriginalFaces(); + } else if constexpr (Lump == BSPLump::GAME_LUMP) { + return this->parseGameLumps(true); + } else { + return this->getLumpData(Lump); + } + } + + /// BSP::setGameLump should be used for writing game lumps as they need special handling. Paklump can be + /// written here but compression is unsupported, prefer using bsppp::PakLump or your favorite zip library + /// instead. Valid compressLevel range is 0 to 9, 0 is considered off, 9 the slowest and most compressiest + bool setLump(BSPLump lumpIndex, uint32_t version, std::span data, uint8_t compressLevel = 0); + + [[nodiscard]] bool isGameLumpCompressed(BSPGameLump::Signature signature) const; + + [[nodiscard]] uint16_t getGameLumpVersion(BSPGameLump::Signature signature); + + [[nodiscard]] std::optional> getGameLumpData(BSPGameLump::Signature signature) const; + + bool setGameLump(BSPGameLump::Signature signature, uint16_t version, std::span data, uint8_t compressLevel = 0); + + /// Reset changes made to a lump before they're written to disk + void resetLump(BSPLump lumpIndex); + + /// Resets ALL in-memory modifications (version, all lumps including game lumps, map revision) + void reset(); + + void createLumpPatchFile(BSPLump lumpIndex) const; + + bool setLumpFromPatchFile(const std::string& lumpFilePath); + + bool bake(std::string_view outputPath = ""); + +protected: + bool readHeader(); + + [[nodiscard]] std::vector parsePlanes() const; + + [[nodiscard]] std::vector parseTextureData() const; + + [[nodiscard]] std::vector parseVertices() const; + + [[nodiscard]] std::vector parseNodes() const; + + [[nodiscard]] std::vector parseTextureInfo() const; + + [[nodiscard]] std::vector parseFaces() const; + + [[nodiscard]] std::vector parseEdges() const; + + [[nodiscard]] std::vector parseSurfEdges() const; + + [[nodiscard]] std::vector parseBrushModels() const; + + [[nodiscard]] std::vector parseOriginalFaces() const; + + [[nodiscard]] std::vector parseGameLumps(bool decompress) const; + + [[nodiscard]] static std::optional> compressLumpData(const std::span data, uint8_t compressLevel = 6); + + [[nodiscard]] static std::optional> decompressLumpData(const std::span data); + + std::string path; + Header header{}; + + uint32_t stagedVersion{}; + std::unordered_map>> stagedLumps; + std::vector stagedGameLumps; + uint32_t stagedMapRevision{}; + + // Slightly different header despite using the same version just to be quirky + bool isL4D2 = false; +}; + +} // namespace bsppp diff --git a/include/bsppp/bsppp.h b/include/bsppp/bsppp.h index 7df7bb59c..05a44b8bf 100644 --- a/include/bsppp/bsppp.h +++ b/include/bsppp/bsppp.h @@ -1,249 +1,10 @@ #pragma once -#include -#include -#include -#include -#include -#include -#include -#include - -#include +/* + * This header is just included so consumers of this library can + * include it the same way as any of the other SourcePP libraries. + */ +#include "BSP.h" #include "LumpData.h" #include "PakLump.h" - -namespace bsppp { - -constexpr auto BSP_SIGNATURE = sourcepp::parser::binary::makeFourCC("VBSP"); -constexpr auto LZMA_VALVE_SIGNATURE = sourcepp::parser::binary::makeFourCC("LZMA"); - -enum class BSPLump : int32_t { - UNKNOWN = -1, - ENTITIES = 0, - PLANES, - TEXDATA, - VERTEXES, - VISIBILITY, - NODES, - TEXINFO, - FACES, - LIGHTING, - OCCLUSION, - LEAFS, - FACEIDS, - EDGES, - SURFEDGES, - MODELS, - WORLDLIGHTS, - LEAFFACES, - LEAFBRUSHES, - BRUSHES, - BRUSHSIDES, - AREAS, - AREAPORTALS, - S2004_PORTALS, - UNUSED0 = S2004_PORTALS, - SL4D2_PROPCOLLISION = S2004_PORTALS, - S2004_CLUSTERS, - UNUSED1 = S2004_CLUSTERS, - SL4D2_PROPHULLS = S2004_CLUSTERS, - S2004_PORTALVERTS, - UNUSED2 = S2004_PORTALVERTS, - SL4D2_PROPHULLVERTS = S2004_PORTALVERTS, - S2004_CLUSTERPORTALS, - UNUSED3 = S2004_CLUSTERPORTALS, - SL4D2_PROPTRIS = S2004_CLUSTERPORTALS, - DISPINFO, - ORIGINALFACES, - PHYSDISP, - PHYSCOLLIDE, - VERTNORMALS, - VERTNORMALINDICES, - S2004_DISP_LIGHTMAP_ALPHAS, - UNUSED4 = S2004_DISP_LIGHTMAP_ALPHAS, - DISP_VERTS, - DISP_LIGHTMAP_SAMPLE_POSITIONS, - GAME_LUMP, - LEAFWATERDATA, - PRIMITIVES, - PRIMVERTS, - PRIMINDICES, - PAKFILE, - CLIPPORTALVERTS, - CUBEMAPS, - TEXDATA_STRING_DATA, - TEXDATA_STRING_TABLE, - OVERLAYS, - LEAFMINDISTTOWATER, - FACE_MACRO_TEXTURE_INFO, - DISP_TRIS, - S2004_PHYSCOLLIDESURFACE, - UNUSED5 = S2004_PHYSCOLLIDESURFACE, - SL4D2_PROP_BLOB = S2004_PHYSCOLLIDESURFACE, - WATEROVERLAYS, - S2006_XBOX_LIGHTMAPPAGES, - LEAF_AMBIENT_INDEX_HDR = S2006_XBOX_LIGHTMAPPAGES, - S2006_XBOX_LIGHTMAPPAGEINFOS, - LEAF_AMBIENT_INDEX = S2006_XBOX_LIGHTMAPPAGEINFOS, - LIGHTING_HDR, - WORLDLIGHTS_HDR, - LEAF_AMBIENT_LIGHTING_HDR, - LEAF_AMBIENT_LIGHTING, - XBOX_XZIPPAKFILE, - FACES_HDR, - MAP_FLAGS, - OVERLAY_FADES, - L4D_OVERLAY_SYSTEM_LEVELS, - UNUSED6 = L4D_OVERLAY_SYSTEM_LEVELS, - L4D2_PHYSLEVEL, - UNUSED7 = L4D2_PHYSLEVEL, - ASW_DISP_MULTIBLEND, - UNUSED8 = ASW_DISP_MULTIBLEND, - - COUNT, -}; -constexpr int32_t BSP_LUMP_COUNT = 64; -static_assert(static_cast>(BSPLump::COUNT) == BSP_LUMP_COUNT, "Incorrect lump count!"); - -class BSP { - struct Lump { - /// Byte offset into file - uint32_t offset; - /// Length of lump data - uint32_t length; - /// Lump format version - uint32_t version; - /// Uncompressed length if lump is compressed, else 0 - uint32_t uncompressedLength; - }; - - struct Header { - /// Version of the BSP file - uint32_t version; - /// Lump metadata - std::array lumps; - /// Map version number - uint32_t mapRevision; - }; - -public: - explicit BSP(std::string path_, bool loadPatchFiles = true); - - explicit operator bool() const; - - static BSP create(std::string path, uint32_t version = 21, uint32_t mapRevision = 0); - - [[nodiscard]] uint32_t getVersion() const; - - void setVersion(uint32_t version); - - [[nodiscard]] uint32_t getMapRevision() const; - - void setMapRevision(uint32_t mapRevision); - - [[nodiscard]] bool hasLump(BSPLump lumpIndex) const; - - [[nodiscard]] bool isLumpCompressed(BSPLump lumpIndex) const; - - [[nodiscard]] uint32_t getLumpVersion(BSPLump lumpIndex) const; - - [[nodiscard]] std::optional> getLumpData(BSPLump lumpIndex, bool noDecompression = false) const; - - template - [[nodiscard]] auto getLumpData() const { - if constexpr (Lump == BSPLump::PLANES) { - return this->parsePlanes(); - } else if constexpr (Lump == BSPLump::TEXDATA) { - return this->parseTextureData(); - } else if constexpr (Lump == BSPLump::VERTEXES) { - return this->parseVertices(); - } else if constexpr (Lump == BSPLump::NODES) { - return this->parseNodes(); - } else if constexpr (Lump == BSPLump::TEXINFO) { - return this->parseTextureInfo(); - } else if constexpr (Lump == BSPLump::FACES) { - return this->parseFaces(); - } else if constexpr (Lump == BSPLump::EDGES) { - return this->parseEdges(); - } else if constexpr (Lump == BSPLump::SURFEDGES) { - return this->parseSurfEdges(); - } else if constexpr (Lump == BSPLump::MODELS) { - return this->parseBrushModels(); - } else if constexpr (Lump == BSPLump::ORIGINALFACES) { - return this->parseOriginalFaces(); - } else if constexpr (Lump == BSPLump::GAME_LUMP) { - return this->parseGameLumps(true); - } else { - return this->getLumpData(Lump); - } - } - - /// BSP::setGameLump should be used for writing game lumps as they need special handling. Paklump can be - /// written here but compression is unsupported, prefer using bsppp::PakLump or your favorite zip library - /// instead. Valid compressLevel range is 0 to 9, 0 is considered off, 9 the slowest and most compressiest - bool setLump(BSPLump lumpIndex, uint32_t version, std::span data, uint8_t compressLevel = 0); - - [[nodiscard]] bool isGameLumpCompressed(BSPGameLump::Signature signature) const; - - [[nodiscard]] uint16_t getGameLumpVersion(BSPGameLump::Signature signature); - - [[nodiscard]] std::optional> getGameLumpData(BSPGameLump::Signature signature) const; - - bool setGameLump(BSPGameLump::Signature signature, uint16_t version, std::span data, uint8_t compressLevel = 0); - - /// Reset changes made to a lump before they're written to disk - void resetLump(BSPLump lumpIndex); - - /// Resets ALL in-memory modifications (version, all lumps including game lumps, map revision) - void reset(); - - void createLumpPatchFile(BSPLump lumpIndex) const; - - bool setLumpFromPatchFile(const std::string& lumpFilePath); - - bool bake(std::string_view outputPath = ""); - -protected: - bool readHeader(); - - [[nodiscard]] std::vector parsePlanes() const; - - [[nodiscard]] std::vector parseTextureData() const; - - [[nodiscard]] std::vector parseVertices() const; - - [[nodiscard]] std::vector parseNodes() const; - - [[nodiscard]] std::vector parseTextureInfo() const; - - [[nodiscard]] std::vector parseFaces() const; - - [[nodiscard]] std::vector parseEdges() const; - - [[nodiscard]] std::vector parseSurfEdges() const; - - [[nodiscard]] std::vector parseBrushModels() const; - - [[nodiscard]] std::vector parseOriginalFaces() const; - - [[nodiscard]] std::vector parseGameLumps(bool decompress) const; - - [[nodiscard]] static std::optional> compressLumpData(const std::span data, uint8_t compressLevel = 6); - - [[nodiscard]] static std::optional> decompressLumpData(const std::span data); - - std::string path; - Header header{}; - - uint32_t stagedVersion{}; - std::unordered_map>> stagedLumps; - std::vector stagedGameLumps; - uint32_t stagedMapRevision{}; - - // Slightly different header despite using the same version just to be quirky - bool isL4D2 = false; -}; - -} // namespace bsppp diff --git a/src/bsppp/bsppp.cpp b/src/bsppp/BSP.cpp similarity index 99% rename from src/bsppp/bsppp.cpp rename to src/bsppp/BSP.cpp index e704bd74a..bb80bbd27 100644 --- a/src/bsppp/bsppp.cpp +++ b/src/bsppp/BSP.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include diff --git a/src/bsppp/PakLump.cpp b/src/bsppp/PakLump.cpp index ed29ca483..8a803bb8d 100644 --- a/src/bsppp/PakLump.cpp +++ b/src/bsppp/PakLump.cpp @@ -2,7 +2,7 @@ #include -#include +#include #include #include #include diff --git a/src/bsppp/_bsppp.cmake b/src/bsppp/_bsppp.cmake index 08d2dbd6f..a8551bf6e 100644 --- a/src/bsppp/_bsppp.cmake +++ b/src/bsppp/_bsppp.cmake @@ -1,11 +1,12 @@ add_pretty_parser(bsppp DEPS liblzma MINIZIP::minizip sourcepp_parser sourcepp::vpkpp PRECOMPILED_HEADERS + "${CMAKE_CURRENT_SOURCE_DIR}/include/bsppp/BSP.h" "${CMAKE_CURRENT_SOURCE_DIR}/include/bsppp/bsppp.h" "${CMAKE_CURRENT_SOURCE_DIR}/include/bsppp/LumpData.h" "${CMAKE_CURRENT_SOURCE_DIR}/include/bsppp/PakLump.h" SOURCES - "${CMAKE_CURRENT_LIST_DIR}/bsppp.cpp" + "${CMAKE_CURRENT_LIST_DIR}/BSP.cpp" "${CMAKE_CURRENT_LIST_DIR}/PakLump.cpp") target_include_directories(bsppp PRIVATE "${xz_SOURCE_DIR}/src/liblzma/api") From d4ddd232c54e8ccee90eb2fec92b74e30a8ba4dc Mon Sep 17 00:00:00 2001 From: craftablescience Date: Mon, 6 Jan 2025 05:32:38 -0500 Subject: [PATCH 4/6] bsppp: add entity lump parser --- include/bsppp/BSP.h | 9 ++- include/bsppp/EntityLump.h | 135 +++++++++++++++++++++++++++++++++ include/sourcepp/parser/Text.h | 10 +-- src/bsppp/BSP.cpp | 76 +++++++++++++++++++ src/bsppp/EntityLump.cpp | 120 +++++++++++++++++++++++++++++ src/bsppp/_bsppp.cmake | 2 + 6 files changed, 345 insertions(+), 7 deletions(-) create mode 100644 include/bsppp/EntityLump.h create mode 100644 src/bsppp/EntityLump.cpp diff --git a/include/bsppp/BSP.h b/include/bsppp/BSP.h index 7df7bb59c..c5e4e6e03 100644 --- a/include/bsppp/BSP.h +++ b/include/bsppp/BSP.h @@ -11,6 +11,7 @@ #include +#include "EntityLump.h" #include "LumpData.h" #include "PakLump.h" @@ -153,7 +154,9 @@ class BSP { template [[nodiscard]] auto getLumpData() const { - if constexpr (Lump == BSPLump::PLANES) { + if constexpr (Lump == BSPLump::ENTITIES) { + return this->parseEntities(); + } else if constexpr (Lump == BSPLump::PLANES) { return this->parsePlanes(); } else if constexpr (Lump == BSPLump::TEXDATA) { return this->parseTextureData(); @@ -185,6 +188,8 @@ class BSP { /// instead. Valid compressLevel range is 0 to 9, 0 is considered off, 9 the slowest and most compressiest bool setLump(BSPLump lumpIndex, uint32_t version, std::span data, uint8_t compressLevel = 0); + bool setLump(uint32_t version, std::span data, uint8_t compressLevel = 0); + [[nodiscard]] bool isGameLumpCompressed(BSPGameLump::Signature signature) const; [[nodiscard]] uint16_t getGameLumpVersion(BSPGameLump::Signature signature); @@ -208,6 +213,8 @@ class BSP { protected: bool readHeader(); + [[nodiscard]] std::vector parseEntities() const; + [[nodiscard]] std::vector parsePlanes() const; [[nodiscard]] std::vector parseTextureData() const; diff --git a/include/bsppp/EntityLump.h b/include/bsppp/EntityLump.h new file mode 100644 index 000000000..65a5679b5 --- /dev/null +++ b/include/bsppp/EntityLump.h @@ -0,0 +1,135 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include +#include + +namespace bsppp { + +template +concept BSPEntityKeyValueType = std::convertible_to + || std::same_as + || std::same_as + || std::same_as + || std::same_as; + +class BSPEntityKeyValues { +public: + class Element { + friend class BSPEntityKeyValues; + + public: + /// Get the key associated with the element + [[nodiscard]] std::string_view getKey() const; + + /// Set the key associated with the element + void setKey(std::string_view key_); + + /// Get the value associated with the element + [[nodiscard]] std::string_view getValue() const; + + /// Get the value associated with the element as the given type + template + [[nodiscard]] V getValue() const { + if constexpr (std::convertible_to) { + return this->value; + } else if constexpr (std::same_as) { + return static_cast(this->getValue()); + } else if constexpr (std::same_as) { + return std::stoi(std::string{this->value}); + } else if constexpr (std::same_as) { + return std::stoll(std::string{this->value}); + } else if constexpr (std::same_as) { + return std::stof(std::string{this->value}); + } + return V{}; + } + + /// Set the value associated with the element + template + void setValue(V value_) { + if constexpr (std::convertible_to) { + this->value = std::string_view{value_}; + } else { + this->setValue(std::to_string(value_)); + } + } + + /// Set the value associated with the element + template + Element& operator=(V value_) { + this->setValue(value_); + return *this; + } + + /// Check if the given element is invalid + [[nodiscard]] bool isInvalid() const; + + protected: + Element() = default; + + static const Element& getInvalid(); + + std::string key; + std::string value; + }; + + BSPEntityKeyValues() = default; + + /// Check if this entity has one or more keyvalues with the given name + [[nodiscard]] bool hasChild(std::string_view childKey) const; + + /// Get the number of keyvalues + [[nodiscard]] uint64_t getKeyValuesCount() const; + + /// Get the number of keyvalues with the given key + [[nodiscard]] uint64_t getKeyValuesCount(std::string_view childKey) const; + + /// Get the keyvalues of the entity + [[nodiscard]] const std::vector& getKeyValues() const; + + /// Get the keyvalue of the entity at the given index + [[nodiscard]] const Element& operator[](unsigned int n) const; + + /// Get the keyvalue of the entity at the given index + [[nodiscard]] Element& operator[](unsigned int n); + + /// Get the first keyvalue of the entity with the given key + [[nodiscard]] const Element& operator[](std::string_view childKey) const; + + /// Get the first keyvalue of the entity with the given key, or create a new keyvalue if it doesn't exist + [[nodiscard]] Element& operator[](std::string_view childKey); + + /// Get the first keyvalue of the entity with the given key + [[nodiscard]] const Element& operator()(std::string_view childKey) const; + + /// Get the first keyvalue of the entity with the given key, or create a new keyvalue if it doesn't exist + [[nodiscard]] Element& operator()(std::string_view childKey); + + /// Get the nth keyvalue of the entity with the given key + [[nodiscard]] const Element& operator()(std::string_view childKey, unsigned int n) const; + + /// Get the nth keyvalue of the element with the given key, or create a new entity if it doesn't exist + [[nodiscard]] Element& operator()(std::string_view childKey, unsigned int n); + + template + Element& addKeyValue(std::string_view key_, V value_ = {}) { + Element elem; + elem.setKey(key_); + elem.setValue(value_); + this->keyvalues.push_back(elem); + return this->keyvalues.back(); + } + + [[nodiscard]] std::string bake(bool useEscapes) const; + +protected: + std::vector keyvalues; +}; + +} // namespace bsppp diff --git a/include/sourcepp/parser/Text.h b/include/sourcepp/parser/Text.h index 72694a590..84f14e5ee 100644 --- a/include/sourcepp/parser/Text.h +++ b/include/sourcepp/parser/Text.h @@ -136,7 +136,7 @@ void eatWhitespaceAndComments(BufferStream& stream, std::string_view singleLineC * @param escapeSequences Characters that will be escaped if a backslash is present before them. To disable escapes, pass an empty map. * @return A view over the string written to the backing stream. */ -[[nodiscard]] std::string_view readStringToBuffer(BufferStream& stream, BufferStream& backing, std::string_view start = DEFAULT_STRING_START, std::string_view end = DEFAULT_STRING_END, const EscapeSequenceMap& escapeSequences = DEFAULT_ESCAPE_SEQUENCES); +std::string_view readStringToBuffer(BufferStream& stream, BufferStream& backing, std::string_view start = DEFAULT_STRING_START, std::string_view end = DEFAULT_STRING_END, const EscapeSequenceMap& escapeSequences = DEFAULT_ESCAPE_SEQUENCES); /** * Read a string starting at the current stream position. @@ -145,7 +145,7 @@ void eatWhitespaceAndComments(BufferStream& stream, std::string_view singleLineC * @param escapeSequences Characters that will be escaped if a backslash is present before them. To disable escapes, pass an empty map. * @return A view over the string written to the backing stream. */ -[[nodiscard]] std::string_view readUnquotedStringToBuffer(BufferStream& stream, BufferStream& backing, const EscapeSequenceMap& escapeSequences = DEFAULT_ESCAPE_SEQUENCES); +std::string_view readUnquotedStringToBuffer(BufferStream& stream, BufferStream& backing, const EscapeSequenceMap& escapeSequences = DEFAULT_ESCAPE_SEQUENCES); /** * Read a string starting at the current stream position. @@ -155,13 +155,11 @@ void eatWhitespaceAndComments(BufferStream& stream, std::string_view singleLineC * @param escapeSequences Characters that will be escaped if a backslash is present before them. To disable escapes, pass an empty map. * @return A view over the string written to the backing stream. */ -[[nodiscard]] std::string_view readUnquotedStringToBuffer(BufferStream& stream, BufferStream& backing, std::string_view end, const EscapeSequenceMap& escapeSequences = DEFAULT_ESCAPE_SEQUENCES); +std::string_view readUnquotedStringToBuffer(BufferStream& stream, BufferStream& backing, std::string_view end, const EscapeSequenceMap& escapeSequences = DEFAULT_ESCAPE_SEQUENCES); class syntax_error : public std::runtime_error { public: - explicit syntax_error(const char* message) noexcept : std::runtime_error(message) {} - syntax_error(const syntax_error& other) noexcept = default; - syntax_error& operator=(const syntax_error& other) noexcept = default; + using std::runtime_error::runtime_error; }; } // namespace sourcepp::parser::text diff --git a/src/bsppp/BSP.cpp b/src/bsppp/BSP.cpp index bb80bbd27..86183993d 100644 --- a/src/bsppp/BSP.cpp +++ b/src/bsppp/BSP.cpp @@ -183,6 +183,22 @@ bool BSP::setLump(BSPLump lumpIndex, uint32_t version, std::span data, uint8_t compressLevel) { + if (version > 1) { + return false; + } + std::string out; + for (const auto& ent : data) { + out += ent.bake(version == 1) + '\n'; + } + if (out.empty()) { + out += '\0'; + } else { + out.back() = '\0'; + } + return this->setLump(BSPLump::ENTITIES, version, {reinterpret_cast(out.data()), out.size()}, compressLevel); +} + bool BSP::isGameLumpCompressed(BSPGameLump::Signature signature) const { for (const auto& gameLump : this->stagedGameLumps) { if (gameLump.signature == signature) { @@ -505,6 +521,66 @@ bool BSP::readHeader() { return true; } +std::vector BSP::parseEntities() const { + using namespace sourcepp; + + const auto useEscapes = this->getLumpVersion(BSPLump::ENTITIES); + if (useEscapes > 1) { + return {}; + } + + auto data = this->getLumpData(BSPLump::ENTITIES); + if (!data) { + return {}; + } + BufferStreamReadOnly stream{*data}; + + std::vector entities; + + try { + while (true) { + // Expect an opening brace + parser::text::eatWhitespaceAndSingleLineComments(stream); + if (stream.peek() != '{') { + break; + } + + auto& ent = entities.emplace_back(); + + while (true) { + // Check if the block is over + parser::text::eatWhitespaceAndSingleLineComments(stream); + if (stream.peek() == '}') { + stream.skip(); + break; + } + + std::string key, value; + + // Read key + { + BufferStream keyStream{key}; + parser::text::readStringToBuffer(stream, keyStream, parser::text::DEFAULT_STRING_START, parser::text::DEFAULT_STRING_END, useEscapes ? parser::text::DEFAULT_ESCAPE_SEQUENCES : parser::text::NO_ESCAPE_SEQUENCES); + key.resize(keyStream.tell()); + parser::text::eatWhitespaceAndSingleLineComments(stream); + } + + // Read value + { + BufferStream valueStream{value}; + parser::text::readStringToBuffer(stream, valueStream, parser::text::DEFAULT_STRING_START, parser::text::DEFAULT_STRING_END, useEscapes ? parser::text::DEFAULT_ESCAPE_SEQUENCES : parser::text::NO_ESCAPE_SEQUENCES); + value.resize(valueStream.tell()); + parser::text::eatWhitespaceAndSingleLineComments(stream); + } + + ent[key] = value; + } + } + } catch (const std::overflow_error&) {} + + return entities; +} + std::vector BSP::parsePlanes() const { return ::parseLumpContents(*this, BSPLump::PLANES); } diff --git a/src/bsppp/EntityLump.cpp b/src/bsppp/EntityLump.cpp new file mode 100644 index 000000000..829e2460e --- /dev/null +++ b/src/bsppp/EntityLump.cpp @@ -0,0 +1,120 @@ +#include + +using namespace bsppp; +using namespace sourcepp; + +std::string_view BSPEntityKeyValues::Element::getKey() const { + return this->key; +} + +void BSPEntityKeyValues::Element::setKey(std::string_view key_) { + this->key = key_; +} + +std::string_view BSPEntityKeyValues::Element::getValue() const { + return this->value; +} + +bool BSPEntityKeyValues::Element::isInvalid() const { + return this == &getInvalid(); +} + +const BSPEntityKeyValues::Element& BSPEntityKeyValues::Element::getInvalid() { + static Element element; + return element; +} + +bool BSPEntityKeyValues::hasChild(std::string_view childKey) const { + return !this->operator[](childKey).isInvalid(); +} + +uint64_t BSPEntityKeyValues::getKeyValuesCount() const { + return this->keyvalues.size(); +} + +uint64_t BSPEntityKeyValues::getKeyValuesCount(std::string_view childKey) const { + uint64_t count = 0; + for (const auto& element : this->keyvalues) { + if (sourcepp::string::iequals(element.getKey(), childKey)) { + ++count; + } + } + return count; +} + +const std::vector& BSPEntityKeyValues::getKeyValues() const { + return this->keyvalues; +} + +const BSPEntityKeyValues::Element& BSPEntityKeyValues::operator[](unsigned int n) const { + return this->keyvalues.at(n); +} + +BSPEntityKeyValues::Element& BSPEntityKeyValues::operator[](unsigned int n) { + return this->keyvalues.at(n); +} + +const BSPEntityKeyValues::Element& BSPEntityKeyValues::operator[](std::string_view childKey) const { + return this->operator()(childKey); +} + +BSPEntityKeyValues::Element& BSPEntityKeyValues::operator[](std::string_view childKey) { + return this->operator()(childKey); +} + +const BSPEntityKeyValues::Element& BSPEntityKeyValues::operator()(std::string_view childKey) const { + for (const auto& element : this->keyvalues) { + if (sourcepp::string::iequals(element.getKey(), childKey)) { + return element; + } + } + return Element::getInvalid(); +} + +BSPEntityKeyValues::Element& BSPEntityKeyValues::operator()(std::string_view childKey) { + for (auto& element : this->keyvalues) { + if (sourcepp::string::iequals(element.getKey(), childKey)) { + return element; + } + } + return this->addKeyValue(childKey); +} + +const BSPEntityKeyValues::Element& BSPEntityKeyValues::operator()(std::string_view childKey, unsigned int n) const { + unsigned int count = 0; + for (const auto& element : this->keyvalues) { + if (sourcepp::string::iequals(element.getKey(), childKey)) { + if (count == n) { + return element; + } + ++count; + } + } + return Element::getInvalid(); +} + +BSPEntityKeyValues::Element& BSPEntityKeyValues::operator()(std::string_view childKey, unsigned int n) { + unsigned int count = 0; + for (auto& element: this->keyvalues) { + if (sourcepp::string::iequals(element.getKey(), childKey)) { + if (count == n) { + return element; + } + ++count; + } + } + return this->addKeyValue(childKey); +} + +std::string BSPEntityKeyValues::bake(bool useEscapes) const { + std::string out = "{\n"; + for (const auto& elem : this->keyvalues) { + out += "\t\""; + out += parser::text::convertSpecialCharsToEscapes(elem.getKey(), useEscapes ? parser::text::DEFAULT_ESCAPE_SEQUENCES : parser::text::NO_ESCAPE_SEQUENCES); + out += "\" \""; + out += parser::text::convertSpecialCharsToEscapes(elem.getValue(), useEscapes ? parser::text::DEFAULT_ESCAPE_SEQUENCES : parser::text::NO_ESCAPE_SEQUENCES); + out += "\"\n"; + } + out += "}"; + return out; +} diff --git a/src/bsppp/_bsppp.cmake b/src/bsppp/_bsppp.cmake index a8551bf6e..1201a7624 100644 --- a/src/bsppp/_bsppp.cmake +++ b/src/bsppp/_bsppp.cmake @@ -3,10 +3,12 @@ add_pretty_parser(bsppp PRECOMPILED_HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/include/bsppp/BSP.h" "${CMAKE_CURRENT_SOURCE_DIR}/include/bsppp/bsppp.h" + "${CMAKE_CURRENT_SOURCE_DIR}/include/bsppp/EntityLump.h" "${CMAKE_CURRENT_SOURCE_DIR}/include/bsppp/LumpData.h" "${CMAKE_CURRENT_SOURCE_DIR}/include/bsppp/PakLump.h" SOURCES "${CMAKE_CURRENT_LIST_DIR}/BSP.cpp" + "${CMAKE_CURRENT_LIST_DIR}/EntityLump.cpp" "${CMAKE_CURRENT_LIST_DIR}/PakLump.cpp") target_include_directories(bsppp PRIVATE "${xz_SOURCE_DIR}/src/liblzma/api") From 5e52688e22f8d51a3fa1923690ebe26921b5f612 Mon Sep 17 00:00:00 2001 From: craftablescience Date: Mon, 6 Jan 2025 20:34:31 -0500 Subject: [PATCH 5/6] bsppp: bsp parsing/writing fixes for bugs that got introduced in the refactors --- include/bsppp/BSP.h | 13 ++++++++ src/bsppp/BSP.cpp | 76 ++++++++++++++++++++++++++++++++---------- src/bsppp/_bsppp.cmake | 3 +- 3 files changed, 73 insertions(+), 19 deletions(-) diff --git a/include/bsppp/BSP.h b/include/bsppp/BSP.h index c5e4e6e03..0a31451e1 100644 --- a/include/bsppp/BSP.h +++ b/include/bsppp/BSP.h @@ -108,6 +108,19 @@ enum class BSPLump : int32_t { constexpr int32_t BSP_LUMP_COUNT = 64; static_assert(static_cast>(BSPLump::COUNT) == BSP_LUMP_COUNT, "Incorrect lump count!"); +/// Pulled from Portal 2, map e1912. This is not a given for every game or even map, and obviously lump order doesn't +/// matter one bit, but we do at least want the paklump to be at the end since it commonly grows and shrinks. +constexpr std::array BSP_LUMP_ORDER{ + 25, 24, 32, 57, 49, 59, 6, 2, + 43, 44, 10, 17, 1, 18, 19, 14, + 5, 20, 21, 4, 0, 29, 26, 62, + 3, 12, 13, 7, 58, 33, 48, 63, + 28, 9, 8, 53, 37, 38, 39, 30, + 31, 56, 52, 51, 55, 16, 36, 45, + 50, 60, 61, 46, 42, 41, 54, 15, + 34, 47, 11, 22, 23, 27, 35, 40, +}; + class BSP { struct Lump { /// Byte offset into file diff --git a/src/bsppp/BSP.cpp b/src/bsppp/BSP.cpp index 86183993d..1846760d7 100644 --- a/src/bsppp/BSP.cpp +++ b/src/bsppp/BSP.cpp @@ -77,9 +77,14 @@ BSP BSP::create(std::string path, uint32_t version, uint32_t mapRevision) { if (version == 27) { writer.write(0); } - writer - .write({}) - .write(mapRevision); + for (int i = 0; i < BSP_LUMP_COUNT; i++) { + writer + .write(0) + .write(0) + .write(0) + .write(0); + } + writer << mapRevision; } return BSP{std::move(path)}; } @@ -300,7 +305,7 @@ void BSP::createLumpPatchFile(BSPLump lumpIndex) const { const auto fsPath = std::filesystem::path{this->path}; const auto fsStem = (fsPath.parent_path() / fsPath.stem()).string() + "_l_"; int nonexistentNumber = 0; - while (1) { + while (true) { if (!std::filesystem::exists(fsStem + std::to_string(nonexistentNumber) + ".lmp")) { break; } @@ -355,26 +360,36 @@ bool BSP::bake(std::string_view outputPath) { } const auto lumpsHeaderOffset = stream.tell(); - stream.write({}); + for (int i = 0; i < sizeof(Header::lumps); i++) { + stream.write(0); + } stream << this->stagedMapRevision; - for (int32_t i = 0; i < BSP_LUMP_COUNT; i++) { + for (auto i : BSP_LUMP_ORDER) { if (!this->hasLump(static_cast(i))) { continue; } + // Lumps are 4 byte aligned + if (const auto padding = math::paddingForAlignment(4, stream.tell()); padding > 0) { + for (int p = 0; p < padding; p++) { + stream.write(0); + } + } + if (static_cast(i) == BSPLump::GAME_LUMP && !this->stagedGameLumps.empty()) { const auto gameLumpOffset = stream.tell(); bool oneOrMoreGameLumpCompressed = false; - for (auto gameLump : this->stagedGameLumps) { + for (const auto& gameLump : this->stagedGameLumps) { if (gameLump.isCompressed) { oneOrMoreGameLumpCompressed = true; break; } } - auto gameLumpCurrentOffset = sizeof(int32_t) + ((sizeof(BSPGameLump) - sizeof(BSPGameLump::data)) * (this->stagedGameLumps.size() + oneOrMoreGameLumpCompressed)); + // NOLINTNEXTLINE(*-sizeof-container) + auto gameLumpCurrentOffset = stream.tell() + sizeof(int32_t) + ((sizeof(BSPGameLump) - sizeof(BSPGameLump::data)) * (this->stagedGameLumps.size() + oneOrMoreGameLumpCompressed)); stream.write(this->stagedGameLumps.size() + oneOrMoreGameLumpCompressed); for (const auto& gameLump : this->stagedGameLumps) { @@ -450,21 +465,33 @@ bool BSP::bake(std::string_view outputPath) { if (data) { const auto curPos = stream.tell(); stream.seek_u(lumpsHeaderOffset + (i * sizeof(Lump))); + + auto& lump = this->header.lumps[i]; if (!this->isL4D2) { stream .write(curPos) - .write(this->header.lumps[i].length) - .write(this->header.lumps[i].version); + .write(lump.length) + .write(lump.version); } else { stream - .write(this->header.lumps[i].version) + .write(lump.version) .write(curPos) - .write(this->header.lumps[i].length); + .write(lump.length); } stream - .write(this->header.lumps[i].uncompressedLength) + .write(lump.uncompressedLength) .seek_u(curPos) .write(*data); + } else { + // We should never be here! + SOURCEPP_DEBUG_BREAK; + } + } + + // Lumps are 4 byte aligned + if (const auto padding = math::paddingForAlignment(4, stream.tell()); padding > 0) { + for (int p = 0; p < padding; p++) { + stream.write(0); } } @@ -496,7 +523,13 @@ bool BSP::readHeader() { reader.skip_in(); } - reader >> this->header.lumps; + for (auto& lump : this->header.lumps) { + reader + >> lump.offset + >> lump.length + >> lump.version + >> lump.uncompressedLength; + } // If no offsets are larger than 1024 (less than the size of the BSP header, but greater // than any lump version), it's probably a L4D2 BSP and those are lump versions! @@ -539,11 +572,17 @@ std::vector BSP::parseEntities() const { try { while (true) { + // Check for EOF - give 3 chars for extra breathing room + if (stream.tell() >= stream.size() - 3) { + return entities; + } + // Expect an opening brace parser::text::eatWhitespaceAndSingleLineComments(stream); if (stream.peek() != '{') { break; } + stream.skip(); auto& ent = entities.emplace_back(); @@ -561,7 +600,7 @@ std::vector BSP::parseEntities() const { { BufferStream keyStream{key}; parser::text::readStringToBuffer(stream, keyStream, parser::text::DEFAULT_STRING_START, parser::text::DEFAULT_STRING_END, useEscapes ? parser::text::DEFAULT_ESCAPE_SEQUENCES : parser::text::NO_ESCAPE_SEQUENCES); - key.resize(keyStream.tell()); + key.resize(keyStream.tell() - 1); parser::text::eatWhitespaceAndSingleLineComments(stream); } @@ -569,15 +608,16 @@ std::vector BSP::parseEntities() const { { BufferStream valueStream{value}; parser::text::readStringToBuffer(stream, valueStream, parser::text::DEFAULT_STRING_START, parser::text::DEFAULT_STRING_END, useEscapes ? parser::text::DEFAULT_ESCAPE_SEQUENCES : parser::text::NO_ESCAPE_SEQUENCES); - value.resize(valueStream.tell()); + value.resize(valueStream.tell() - 1); parser::text::eatWhitespaceAndSingleLineComments(stream); } ent[key] = value; } } - } catch (const std::overflow_error&) {} - + } catch (const std::overflow_error&) { + return {}; + } return entities; } diff --git a/src/bsppp/_bsppp.cmake b/src/bsppp/_bsppp.cmake index 1201a7624..dea095906 100644 --- a/src/bsppp/_bsppp.cmake +++ b/src/bsppp/_bsppp.cmake @@ -1,5 +1,6 @@ add_pretty_parser(bsppp - DEPS liblzma MINIZIP::minizip sourcepp_parser sourcepp::vpkpp + DEPS liblzma MINIZIP::minizip sourcepp_parser + DEPS_PUBLIC sourcepp::vpkpp PRECOMPILED_HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/include/bsppp/BSP.h" "${CMAKE_CURRENT_SOURCE_DIR}/include/bsppp/bsppp.h" From f600d34bf66a0dcaf7729497b4d3ed21c6fb96b8 Mon Sep 17 00:00:00 2001 From: craftablescience Date: Mon, 6 Jan 2025 05:33:05 -0500 Subject: [PATCH 6/6] kvpp: support reading hex strings --- include/kvpp/KV1.h | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/include/kvpp/KV1.h b/include/kvpp/KV1.h index 378bd45fe..3238d2457 100644 --- a/include/kvpp/KV1.h +++ b/include/kvpp/KV1.h @@ -15,7 +15,8 @@ namespace kvpp { template concept KV1ValueType = std::convertible_to || std::same_as - || std::same_as + || std::same_as + || std::same_as || std::same_as; template @@ -38,9 +39,17 @@ class KV1ElementBase { if constexpr (std::convertible_to) { return this->value; } else if constexpr (std::same_as) { - return static_cast(this->getValue()); - } else if constexpr (std::same_as) { + return static_cast(this->getValue()); + } else if constexpr (std::same_as) { + if (this->value.length() == 10 && this->value.starts_with("0x") && sourcepp::parser::text::isNumber(this->value.substr(2))) { + return std::stoi(std::string{this->value.substr(2)}, nullptr, 16); + } return std::stoi(std::string{this->value}); + } else if constexpr (std::same_as) { + if (this->value.length() == 18 && this->value.starts_with("0x") && sourcepp::parser::text::isNumber(this->value.substr(2))) { + return std::stoll(std::string{this->value.substr(2)}, nullptr, 16); + } + return std::stoll(std::string{this->value}); } else if constexpr (std::same_as) { return std::stof(std::string{this->value}); } @@ -206,7 +215,7 @@ class KV1ElementWritable : public KV1ElementBase> { void setValue(V value_) { if constexpr (std::convertible_to) { this->value = std::string_view{value_}; - } else if constexpr (std::same_as || std::same_as || std::same_as) { + } else { this->setValue(std::to_string(value_)); } }