Skip to content
This repository has been archived by the owner on Aug 4, 2023. It is now read-only.

Commit

Permalink
No more flat groups, but something that is even faster
Browse files Browse the repository at this point in the history
  • Loading branch information
@xforce
xforce committed Jun 14, 2019
1 parent 1b238bb commit fad7994
Show file tree
Hide file tree
Showing 4 changed files with 75 additions and 144 deletions.
60 changes: 18 additions & 42 deletions cmd/xmltest/src/main.cc
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
#include "xml_operations.h"

#include "absl/strings/str_cat.h"
#include "pugixml.hpp"
#include "spdlog/spdlog.h"

#include <cstdio>
#include <cstring>
Expand All @@ -26,59 +28,33 @@ int main(int argc, const char **argv)
doc->load_buffer(buffer.data(), buffer.size());
}

// Flatten shit
std::vector<pugi::xml_node> assets;
auto nodes = doc->select_nodes("//Assets");
for (pugi::xpath_node node : nodes) {
for (pugi::xml_node asset : node.node().children()) {
assets.push_back(asset);
}
}

auto new_doc = std::make_shared<pugi::xml_document>();
auto groups = new_doc->root().append_child("AssetList").append_child("Groups");
auto assets_xml = groups.append_child("Group").append_child("Assets");
for (pugi::xml_node asset : assets) {
auto guid = asset.parent().parent().find_child(
[](pugi::xml_node x) { return x.name() == std::string("GUID"); });
if (!guid) {
assets_xml.append_copy(asset);
} else {
// Check if we have already created this one
std::string guid_str = guid.first_child().value();
auto group = new_doc->select_nodes(
("/AssetList/Groups/Group[GUID='" + guid_str + "']/Assets").c_str());
pugi::xml_node group_xml;
if (std::begin(group) == std::end(group)) {
// No group create new
group_xml = groups.append_child("Group");
group_xml.append_child("GUID")
.append_child(pugi::xml_node_type::node_pcdata)
.set_value(guid_str.c_str());
group_xml = group_xml.append_child("Assets");
} else {
group_xml = std::begin(group)->node();
}
group_xml.append_copy(asset);
}
}

auto operations = XmlOperation::GetXmlOperationsFromFile(argv[2]);
for (auto &&operation : operations) {
operation.Apply(new_doc);
operation.Apply(doc);
}

std::stringstream ss;
new_doc->print(ss);
struct xml_string_writer : pugi::xml_writer {
std::string result;

virtual void write(const void *data, size_t size)
{
absl::StrAppend(&result, std::string_view{(const char *)data, size});
}
};

xml_string_writer writer;
spdlog::info("Start writing");
writer.result.reserve(100 * 1024 * 1024);
doc->print(writer);
spdlog::info("Finished writing");
FILE *fp;
fp = fopen("patched.xml", "w+");
if (!fp) {
printf("Could not open file for writing\n");

return 0;
}
std::string buf = ss.str();
fwrite(buf.data(), 1, buf.size(), fp);
fwrite(writer.result.data(), 1, writer.result.size(), fp);
fclose(fp);
return 0;
}
5 changes: 2 additions & 3 deletions libs/external-file-loader/src/external-file-loader.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,11 @@ uintptr_t* ReadFileFromContainerOIP = nullptr;
bool __fastcall ReadFileFromContainer(__int64 archive_file_map, const std::wstring& file_path,
char** output_data_pointer, size_t* output_data_size)
{
auto m = file_path;
// archive_file_map is a pointer to a struct identifying which rda this file resides in
// as each rda is actually just a memory mapped file
// but we don't care about that at the moment, probably never will
if (ModManager::instance().IsFileModded(m)) {
auto info = ModManager::instance().GetModdedFileInfo(m);
if (ModManager::instance().IsFileModded(file_path)) {
auto info = ModManager::instance().GetModdedFileInfo(file_path);
if (info.is_patched) {
memcpy(*output_data_pointer, info.data.data(), info.data.size());
} else {
Expand Down
75 changes: 2 additions & 73 deletions libs/external-file-loader/src/mod_manager.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
#include <optional>
#include <sstream>

constexpr static auto PATCH_OP_VERSION = "1.3";
constexpr static auto PATCH_OP_VERSION = "1.4";

Mod& ModManager::Create(const fs::path& root)
{
Expand Down Expand Up @@ -387,78 +387,6 @@ void ModManager::GameFilesReady()
std::string last_valid_cache = "";
std::string next_input_hash = game_file_hash;

if (game_path.wstring().find(L"assets.xml") != std::wstring::npos) {
const auto output_hash = CheckCacheLayer(game_path, next_input_hash, "cookie");
if (output_hash) {
// Cache hit
last_valid_cache = *output_hash;
next_input_hash = *output_hash;
} else {
game_xml = std::make_shared<pugi::xml_document>();
auto parse_result = game_xml->load_buffer(game_file.data(), game_file.size());
if (!parse_result) {
spdlog::error("Failed to parse {}: {}", game_path.string(),
parse_result.description());
}

std::vector<pugi::xml_node> assets;
auto nodes = game_xml->select_nodes("//Assets");
for (pugi::xpath_node node : nodes) {
for (pugi::xml_node asset : node.node().children()) {
assets.push_back(asset);
}
}

auto new_doc = std::make_shared<pugi::xml_document>();
auto groups = new_doc->root().append_child("AssetList").append_child("Groups");
auto assets_xml = groups.append_child("Group").append_child("Assets");
for (pugi::xml_node asset : assets) {
auto guid = asset.parent().parent().find_child(
[](pugi::xml_node x) { return x.name() == std::string("GUID"); });
if (!guid) {
assets_xml.append_copy(asset);
} else {
// Check if we have already created this one
std::string guid_str = guid.first_child().value();
auto group = new_doc->select_nodes(
("/AssetList/Groups/Group[GUID='" + guid_str + "']/Assets")
.c_str());
pugi::xml_node group_xml;
if (std::begin(group) == std::end(group)) {
// No group create new
group_xml = groups.append_child("Group");
group_xml.append_child("GUID")
.append_child(pugi::xml_node_type::node_pcdata)
.set_value(guid_str.c_str());
group_xml = group_xml.append_child("Assets");
} else {
group_xml = std::begin(group)->node();
}
group_xml.append_copy(asset);
}
}

struct xml_string_writer : pugi::xml_writer {
std::string result;

virtual void write(const void* data, size_t size)
{
absl::StrAppend(&result, std::string_view{(const char*)data, size});
}
};

xml_string_writer writer;
new_doc->print(writer);
if (last_valid_cache.empty()) {
last_valid_cache = game_file_hash;
}
last_valid_cache = PushCacheLayer(game_path, last_valid_cache, "cookie",
writer.result, "internal");
next_input_hash = last_valid_cache;
game_xml = nullptr;
}
}

for (auto&& on_disk_file : on_disk_files) {
if (shuttding_down_.load()) {
return;
Expand Down Expand Up @@ -505,6 +433,7 @@ void ModManager::GameFilesReady()
};

xml_string_writer writer;
writer.result.reserve(100 * 1024 * 1024);
game_xml->print(writer);
std::string& buf = writer.result;

Expand Down
79 changes: 53 additions & 26 deletions libs/xml-operations/src/xml_operations.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@ XmlOperation::XmlOperation(std::shared_ptr<pugi::xml_document> doc, pugi::xml_no
XmlOperation::XmlOperation(std::shared_ptr<pugi::xml_document> doc, pugi::xml_node node,
std::string guid)
{
doc_ = doc;
doc_ = doc;
guid_ = guid;
ReadPath(node, guid);
ReadType(node);
if (type_ != Type::Remove) {
Expand All @@ -28,43 +29,36 @@ XmlOperation::XmlOperation(std::shared_ptr<pugi::xml_document> doc, pugi::xml_no

void XmlOperation::ReadPath(pugi::xml_node node, std::string guid)
{
auto prop_path = GetXmlPropString(node, "Path");
if (!guid.empty()) {
path_ = "/AssetList/Groups/Group/Assets/Asset[Values/Standard/GUID='" + guid + "']";
speculative_path_ =
"/AssetList/Groups/Group[1]/Assets/Asset[Values/Standard/GUID='" + guid + "']";
path_ = "//Asset[Values/Standard/GUID='" + guid + "']";
}
auto prop_path = GetXmlPropString(node, "Path");
if (prop_path.find("/") != 0) {
path_ += "/";
speculative_path_ += "/";
}
path_ += GetXmlPropString(node, "Path");
speculative_path_ += GetXmlPropString(node, "Path");
path_ += prop_path;
if (path_ == "/") {
path_ = "/*";
}
if (speculative_path_ == "/") {
speculative_path_ = "/*";
}
if (path_.length() > 0) {
if (path_[path_.length() - 1] == '/') {
path_ = path_.substr(0, path_.length() - 1);
}
}
if (speculative_path_.length() > 0) {
if (speculative_path_[speculative_path_.length() - 1] == '/') {
speculative_path_ = speculative_path_.substr(0, speculative_path_.length() - 1);
}
}

if (path_.find("//Assets[") == 0) {
auto npath = path_.substr(strlen("//Assets["));
path_ = "/AssetList/Groups/Group/Assets[" + npath;
speculative_path_ = "/AssetList/Groups/Group[1]/Assets[" + npath;
} else if (path_.find("//Asset") == 0) {
auto npath = path_.substr(strlen("//Asset"));
path_ = "/AssetList/Groups/Group/Assets/Asset" + npath;
speculative_path_ = "/AssetList/Groups/Group[1]/Assets/Asset" + npath;
if (!guid.empty()) {
speculative_path_ += prop_path;
if (speculative_path_ == "/") {
speculative_path_ = "/*";
}
if (speculative_path_.length() > 0) {
if (speculative_path_[path_.length() - 1] == '/') {
speculative_path_ = speculative_path_.substr(0, speculative_path_.length() - 1);
}
}
if (speculative_path_.find("/") == 0) {
speculative_path_ = speculative_path_.substr(1);
}
}
}

Expand All @@ -88,12 +82,44 @@ void XmlOperation::ReadType(pugi::xml_node node)
}
}

std::optional<pugi::xml_node> FindAsset(std::string guid, pugi::xml_node node)
{
//
if (node.name() == std::string("Asset")) {
pugi::xml_node g = node.first_element_by_path("Values/Standard/GUID");
if (g.text().get() == guid) {
return node;
}
return {};
}

for (pugi::xml_node n : node.children()) {
if (auto found = FindAsset(guid, n); found) {
return found;
}
}

return {};
}

std::optional<pugi::xml_node> FindAsset(std::shared_ptr<pugi::xml_document> doc, std::string guid)
{
return FindAsset(guid, doc->root());
}

void XmlOperation::Apply(std::shared_ptr<pugi::xml_document> doc, fs::path mod_path)
{
try {
spdlog::info("Looking up {}", path_);
pugi::xpath_node_set results;
if (!speculative_path_.empty()) {
results = doc->select_nodes(speculative_path_.c_str());
if (!guid_.empty()) {
auto node = FindAsset(doc, guid_);
if (node) {
results = node->select_nodes(speculative_path_.c_str());
}
if (results.empty()) {
spdlog::debug("Speculative path failed to find node {}", GetPath());
}
}
if (results.empty()) {
results = doc->select_nodes(GetPath().c_str());
Expand All @@ -102,6 +128,7 @@ void XmlOperation::Apply(std::shared_ptr<pugi::xml_document> doc, fs::path mod_p
spdlog::warn("No matching node for Path {}", GetPath());
return;
}
spdlog::info("Looking finished {}", path_);
for (pugi::xpath_node xnode : results) {
pugi::xml_node game_node = xnode.node();
if (GetType() == XmlOperation::Type::Merge) {
Expand Down

0 comments on commit fad7994

Please sign in to comment.