VOXELFORMAT: refactored interface to work with archives and not on st…

…reams this allows us to support multi file formats in a better way Fixes issue #460
vengi-voxel · May 13, 2024 · 202c188 · 202c188
1 parent 496b5a9
commit 202c188
Show file tree

Hide file tree

Showing 123 changed files with 2,149 additions and 1,775 deletions.
diff --git a/src/modules/io/Archive.cpp b/src/modules/io/Archive.cpp
@@ -36,27 +36,11 @@ bool isSupportedArchive(const core::String &filename) {
 }
 
 ArchivePtr openArchive(const io::FilesystemPtr &fs, const core::String &path, io::SeekableReadStream *stream) {
-	if (fs->isReadableDir(path)) {
-		auto archive = core::make_shared<FilesystemArchive>(fs);
-		if (!archive->init(path, stream)) {
-			return ArchivePtr{};
-		}
-		return archive;
+	const core::String &ext = core::string::extractExtension(path);
+	if (ext == "zip" || ext == "pk3" || ext == "thing") {
+		return openZipArchive(stream);
 	}
-	const core::String ext = core::string::extractExtension(path);
-	if (ext == "zip" || ext == "pk3" || ext == "thing" || (stream != nullptr && ZipArchive::validStream(*stream))) {
-		auto archive = core::make_shared<ZipArchive>();
-		if (!archive->init(path, stream)) {
-			return ArchivePtr{};
-		}
-		return archive;
-	}
-	const core::String &directory = core::string::extractPath(path);
-	auto archive = core::make_shared<FilesystemArchive>(fs);
-	if (!archive->init(directory, stream)) {
-		return ArchivePtr{};
-	}
-	return archive;
+	return openFilesystemArchive(fs, path);
 }
 
 } // namespace io
diff --git a/src/modules/io/Archive.h b/src/modules/io/Archive.h
@@ -14,8 +14,6 @@ namespace io {
 class Filesystem;
 
 using ArchiveFiles = core::DynamicArray<io::FilesystemEntry>;
-using SeekableReadStreamPtr = core::SharedPtr<SeekableReadStream>;
-using SeekableWriteStreamPtr = core::SharedPtr<SeekableWriteStream>;
 using FilesystemPtr = core::SharedPtr<Filesystem>;
 
 /**

diff --git a/src/modules/io/FilesystemArchive.cpp b/src/modules/io/FilesystemArchive.cpp
@@ -4,6 +4,7 @@
 
 #include "FilesystemArchive.h"
 #include "core/Log.h"
+#include "core/SharedPtr.h"
 #include "core/StringUtil.h"
 #include "io/File.h"
 #include "io/FileStream.h"
@@ -47,6 +48,7 @@ static FileMode convertMode(const core::String &path, FileMode mode) {
 }
 
 io::FilePtr FilesystemArchive::open(const core::String &path, FileMode mode) const {
+	Log::debug("searching for %s in %i files", path.c_str(), (int)_files.size());
 	for (const auto &e : _files) {
 		// TODO: implement case insensitive search
 		if (core::string::endsWith(e.fullPath, path)) {
@@ -69,8 +71,17 @@ io::FilePtr FilesystemArchive::open(const core::String &path, FileMode mode) con
 	return {};
 }
 
-bool FilesystemArchive::exists(const core::String &filePath) const {
-	return open(filePath, FileMode::Read);
+bool FilesystemArchive::exists(const core::String &path) const {
+	for (const auto &e : _files) {
+		// TODO: implement case insensitive search
+		if (core::string::endsWith(e.fullPath, path)) {
+			return true;
+		}
+	}
+	if (_files.empty()) {
+		return _filesytem->exists(path);
+	}
+	return false;
 }
 
 SeekableReadStream* FilesystemArchive::readStream(const core::String &filePath) {
@@ -91,4 +102,12 @@ SeekableWriteStream* FilesystemArchive::writeStream(const core::String &filePath
 	return stream;
 }
 
+ArchivePtr openFilesystemArchive(const io::FilesystemPtr &fs, const core::String &path) {
+	core::SharedPtr<FilesystemArchive> fa = core::make_shared<FilesystemArchive>(fs);
+	if (!path.empty() && fs->isReadableDir(path)) {
+		fa->init(path);
+	}
+	return fa;
+}
+
 } // namespace io
diff --git a/src/modules/io/FilesystemArchive.h b/src/modules/io/FilesystemArchive.h
@@ -28,4 +28,6 @@ class FilesystemArchive : public Archive {
 	SeekableWriteStream* writeStream(const core::String &filePath) override;
 };
 
+ArchivePtr openFilesystemArchive(const io::FilesystemPtr &fs, const core::String &path = "");
+
 } // namespace io
diff --git a/src/modules/io/MemoryArchive.cpp b/src/modules/io/MemoryArchive.cpp
@@ -58,6 +58,7 @@ SeekableReadStream *MemoryArchive::readStream(const core::String &filePath) {
 	if (iter == _entries.end()) {
 		return nullptr;
 	}
+	iter->second->seek(0);
 	return new SeekableReadWriteStreamWrapper((io::SeekableReadStream*)iter->second);
 }
 

diff --git a/src/modules/io/MemoryArchive.h b/src/modules/io/MemoryArchive.h
@@ -4,14 +4,13 @@
 
 #pragma once
 
+#include "core/SharedPtr.h"
 #include "core/collection/StringMap.h"
 #include "io/Archive.h"
-#include "io/BufferedReadWriteStream.h"
-#include "core/SharedPtr.h"
 
 namespace io {
 
-using BufferedReadWriteStreamPtr = core::SharedPtr<BufferedReadWriteStream>;
+class BufferedReadWriteStream;
 
 /**
  * Archive that stores files in memory.
@@ -34,4 +33,10 @@ class MemoryArchive : public Archive {
 	SeekableWriteStream* writeStream(const core::String &filePath) override;
 };
 
+using MemoryArchivePtr = core::SharedPtr<MemoryArchive>;
+
+inline MemoryArchivePtr openMemoryArchive() {
+	return core::make_shared<MemoryArchive>();
+}
+
 } // namespace io
diff --git a/src/modules/io/StreamArchive.h b/src/modules/io/StreamArchive.h
@@ -0,0 +1,32 @@
+/**
+ * @file
+ */
+
+#include "io/Archive.h"
+#include "io/Stream.h"
+
+namespace io {
+
+/**
+ * @ingroup IO
+ */
+class StreamArchive : public Archive {
+private:
+	io::SeekableReadStream *_readStream;
+	io::SeekableWriteStream *_writeStream;
+
+public:
+	StreamArchive(io::SeekableReadStream *stream) : _readStream(stream) {
+	}
+	StreamArchive(io::SeekableWriteStream *stream) : _writeStream(stream) {
+	}
+	~StreamArchive() override = default;
+	SeekableReadStream *readStream(const core::String &filePath) override {
+		return _readStream;
+	}
+	SeekableWriteStream *writeStream(const core::String &filePath) override {
+		return _writeStream;
+	}
+};
+
+} // namespace io
diff --git a/src/modules/io/ZipArchive.cpp b/src/modules/io/ZipArchive.cpp
@@ -4,6 +4,7 @@
 
 #include "ZipArchive.h"
 #include "core/Log.h"
+#include "core/SharedPtr.h"
 #include "core/StandardLib.h"
 #include "core/StringUtil.h"
 #include "io/BufferedReadWriteStream.h"
@@ -174,4 +175,13 @@ SeekableWriteStream* ZipArchive::writeStream(const core::String &filePath) {
 	return nullptr;
 }
 
+ArchivePtr openZipArchive(io::SeekableReadStream *stream) {
+	if (!stream || !ZipArchive::validStream(*stream)) {
+		return ArchivePtr{};
+	}
+	core::SharedPtr<ZipArchive> za = core::make_shared<ZipArchive>();
+	za->init("", stream);
+	return za;
+}
+
 } // namespace io
diff --git a/src/modules/io/ZipArchive.h b/src/modules/io/ZipArchive.h
@@ -29,4 +29,6 @@ class ZipArchive : public Archive {
 	void shutdown() override;
 };
 
+ArchivePtr openZipArchive(io::SeekableReadStream *stream);
+
 } // namespace io
diff --git a/src/modules/voxelcollection/CollectionManager.cpp b/src/modules/voxelcollection/CollectionManager.cpp
@@ -7,6 +7,8 @@
 #include "core/Log.h"
 #include "core/StringUtil.h"
 #include "http/HttpCacheStream.h"
+#include "io/Archive.h"
+#include "io/FilesystemArchive.h"
 #include "voxelcollection/Downloader.h"
 #include "voxelformat/VolumeFormat.h"
 
@@ -126,8 +128,9 @@ void CollectionManager::loadThumbnail(const VoxelFile &voxelFile) {
 				return;
 			}
 			http::HttpCacheStream stream(_filesystem, voxelFile.fullPath, voxelFile.url);
+			const io::ArchivePtr &archive = io::openFilesystemArchive(_filesystem);
 			voxelformat::LoadContext loadCtx;
-			image::ImagePtr thumbnailImage = voxelformat::loadScreenshot(voxelFile.fullPath, stream, loadCtx);
+			image::ImagePtr thumbnailImage = voxelformat::loadScreenshot(voxelFile.fullPath, archive, loadCtx);
 			if (!thumbnailImage || !thumbnailImage->isLoaded()) {
 				Log::debug("Failed to load given input file: %s", voxelFile.fullPath.c_str());
 				return;

diff --git a/src/modules/voxelformat/Format.cpp b/src/modules/voxelformat/Format.cpp
@@ -12,6 +12,7 @@
 #include "core/Var.h"
 #include "core/collection/DynamicArray.h"
 #include "image/Image.h"
+#include "io/Archive.h"
 #include "math/Math.h"
 #include "scenegraph/SceneGraph.h"
 #include "scenegraph/SceneGraphNode.h"
@@ -86,18 +87,18 @@ void Format::calcMinsMaxs(const voxel::Region &region, const glm::ivec3 &maxSize
 	Log::debug("maxs(%i:%i:%i)", maxs.x, maxs.y, maxs.z);
 }
 
-size_t Format::loadPalette(const core::String &, io::SeekableReadStream &, palette::Palette &, const LoadContext &) {
+size_t Format::loadPalette(const core::String &, const io::ArchivePtr &, palette::Palette &, const LoadContext &) {
 	return 0;
 }
 
-size_t PaletteFormat::loadPalette(const core::String &filename, io::SeekableReadStream &stream, palette::Palette &palette,
+size_t PaletteFormat::loadPalette(const core::String &filename, const io::ArchivePtr &archive, palette::Palette &palette,
 								  const LoadContext &ctx) {
 	scenegraph::SceneGraph sceneGraph;
-	loadGroupsPalette(filename, stream, sceneGraph, palette, ctx);
+	loadGroupsPalette(filename, archive, sceneGraph, palette, ctx);
 	return palette.size();
 }
 
-image::ImagePtr Format::loadScreenshot(const core::String &filename, io::SeekableReadStream &, const LoadContext &) {
+image::ImagePtr Format::loadScreenshot(const core::String &filename, const io::ArchivePtr &, const LoadContext &) {
 	Log::debug("%s doesn't have a supported embedded screenshot", filename.c_str());
 	return image::ImagePtr();
 }
@@ -111,7 +112,7 @@ bool Format::singleVolume() const {
 }
 
 bool Format::save(const scenegraph::SceneGraph &sceneGraph, const core::String &filename,
-				  io::SeekableWriteStream &stream, const SaveContext &ctx) {
+				  const io::ArchivePtr &archive, const SaveContext &ctx) {
 	bool needsSplit = false;
 	const glm::ivec3 maxsize = maxSize();
 	if (maxsize.x > 0 && maxsize.y > 0 && maxsize.z > 0) {
@@ -143,14 +144,14 @@ bool Format::save(const scenegraph::SceneGraph &sceneGraph, const core::String &
 		mergedNode.setVolume(merged.first, true);
 		mergedNode.setPalette(merged.second);
 		mergedSceneGraph.emplace(core::move(mergedNode));
-		return saveGroups(mergedSceneGraph, filename, stream, ctx);
+		return saveGroups(mergedSceneGraph, filename, archive, ctx);
 	}
 
 	if (needsSplit) {
 		Log::debug("Split volumes before saving as the target format only supports smaller volume sizes");
 		scenegraph::SceneGraph newSceneGraph;
 		scenegraph::splitVolumes(sceneGraph, newSceneGraph, false, false, saveVisibleOnly, maxsize);
-		return saveGroups(newSceneGraph, filename, stream, ctx);
+		return saveGroups(newSceneGraph, filename, archive, ctx);
 	}
 
 	if (saveVisibleOnly) {
@@ -166,14 +167,14 @@ bool Format::save(const scenegraph::SceneGraph &sceneGraph, const core::String &
 		for (int nodeId : nodes) {
 			newSceneGraph.removeNode(nodeId, false);
 		}
-		return saveGroups(newSceneGraph, filename, stream, ctx);
+		return saveGroups(newSceneGraph, filename, archive, ctx);
 	}
-	return saveGroups(sceneGraph, filename, stream, ctx);
+	return saveGroups(sceneGraph, filename, archive, ctx);
 }
 
-bool Format::load(const core::String &filename, io::SeekableReadStream &stream, scenegraph::SceneGraph &sceneGraph,
+bool Format::load(const core::String &filename, const io::ArchivePtr &archive, scenegraph::SceneGraph &sceneGraph,
 				  const LoadContext &ctx) {
-	if (!loadGroups(filename, stream, sceneGraph, ctx)) {
+	if (!loadGroups(filename, archive, sceneGraph, ctx)) {
 		return false;
 	}
 	if (!sceneGraph.validate()) {
@@ -191,18 +192,18 @@ bool Format::stopExecution() {
 	return app::App::getInstance()->shouldQuit();
 }
 
-bool PaletteFormat::loadGroups(const core::String &filename, io::SeekableReadStream &stream,
+bool PaletteFormat::loadGroups(const core::String &filename, const io::ArchivePtr &archive,
 							   scenegraph::SceneGraph &sceneGraph, const LoadContext &ctx) {
 	palette::Palette palette;
-	if (!loadGroupsPalette(filename, stream, sceneGraph, palette, ctx)) {
+	if (!loadGroupsPalette(filename, archive, sceneGraph, palette, ctx)) {
 		return false;
 	}
 	sceneGraph.updateTransforms();
 	return true;
 }
 
 bool PaletteFormat::save(const scenegraph::SceneGraph &sceneGraph, const core::String &filename,
-						 io::SeekableWriteStream &stream, const SaveContext &ctx) {
+						 const io::ArchivePtr &archive, const SaveContext &ctx) {
 	if (onlyOnePalette() && sceneGraph.hasMoreThanOnePalette()) {
 		Log::debug("Need to merge palettes before saving");
 		const palette::Palette &palette = sceneGraph.mergePalettes(true, emptyPaletteIndex());
@@ -213,7 +214,7 @@ bool PaletteFormat::save(const scenegraph::SceneGraph &sceneGraph, const core::S
 			node.remapToPalette(palette);
 			node.setPalette(palette);
 		}
-		return Format::save(newSceneGraph, filename, stream, ctx);
+		return Format::save(newSceneGraph, filename, archive, ctx);
 	} else if (emptyPaletteIndex() >= 0 && emptyPaletteIndex() < palette::PaletteMaxColors) {
 		Log::debug("Need to convert voxels to a palette that has %i as an empty slot", emptyPaletteIndex());
 		scenegraph::SceneGraph newSceneGraph;
@@ -258,9 +259,9 @@ bool PaletteFormat::save(const scenegraph::SceneGraph &sceneGraph, const core::S
 				node.remapToPalette(node.palette(), emptyPaletteIndex());
 			}
 		}
-		return Format::save(newSceneGraph, filename, stream, ctx);
+		return Format::save(newSceneGraph, filename, archive, ctx);
 	}
-	return Format::save(sceneGraph, filename, stream, ctx);
+	return Format::save(sceneGraph, filename, archive, ctx);
 }
 
 Format::Format() {
@@ -275,20 +276,18 @@ core::RGBA Format::flattenRGB(uint8_t r, uint8_t g, uint8_t b, uint8_t a) const
 	return core::Color::flattenRGB(r, g, b, a, _flattenFactor);
 }
 
-bool RGBAFormat::loadGroups(const core::String &filename, io::SeekableReadStream &stream,
+bool RGBAFormat::loadGroups(const core::String &filename, const io::ArchivePtr &archive,
 							scenegraph::SceneGraph &sceneGraph, const LoadContext &ctx) {
 	palette::Palette palette;
 	const bool createPalette = core::Var::get(cfg::VoxelCreatePalette);
 	if (createPalette) {
-		const int64_t resetToPos = stream.pos();
-		if (loadPalette(filename, stream, palette, ctx) <= 0) {
+		if (loadPalette(filename, archive, palette, ctx) <= 0) {
 			palette = voxel::getPalette();
 		}
-		stream.seek(resetToPos);
 	} else {
 		palette = voxel::getPalette();
 	}
-	if (!loadGroupsRGBA(filename, stream, sceneGraph, palette, ctx)) {
+	if (!loadGroupsRGBA(filename, archive, sceneGraph, palette, ctx)) {
 		return false;
 	}
 	sceneGraph.updateTransforms();