State state handling updates and optimizations

* Imagine: Add DynArray class
* Imagine: Add Gzip utility functions
* Imagine: Allow rewinding ArchiveIterator after incrementing to the end
* Imagine: Add IOBuffer constructor to IO and span constructor to MapIO
* Imagine: Add PosixIO and MapIO constructors to PosixFileIO
* Imagine: Rename IOUtils.hh to IOUtils-impl.hh and make it an external header
* EmuFramework: Add new readState()/writeState() API for working with states in memory buffers, needed to support future features like rewinding
* EmuFramework: Cache FileIO for autosave states to reduce overhead when writing data mid-emulation
* EmuFramework: Add OutSizeTracker class
* EmuFramework: Only consider autosave state older than backup memory if the timestamps differ by more than 1 second
* C64.emu: Cache the system files archive iterator to speed up successive calls to sysfile_locate() and sysfile_load()
* NEO.emu: Fix crash when opening content list menu and the content folder doesn't exist
* Convert all systems to new save state API

2600.emu/src/main/Main.cc

@@ -136,6 +136,9 @@ void A2600System::loadContent(IO &io, EmuSystemCreateParams, OnLoadProgressDeleg
 	console.initializeVideo();
 	console.initializeAudio();
 	logMsg("is PAL: %s", videoSystem() == VideoSystem::PAL ? "yes" : "no");
+	Serializer state;
+	osystem.state().saveState(state);
+	saveStateSize = state.size();
 }
 
 static auto consoleFrameRate(const OSystem &osystem)
@@ -222,23 +225,23 @@ void A2600System::reset(EmuApp &, ResetMode mode)
 	}
 }
 
-void A2600System::saveState(IG::CStringView path)
+void A2600System::readState(EmuApp &app, std::span<uint8_t> buff)
 {
-	Serializer state{path.data()};
-	if(!osystem.state().saveState(state))
-	{
-		throwFileWriteError();
-	}
+	Serializer state;
+	state.putByteArray(buff.data(), buff.size());
+	if(!osystem.state().loadState(state))
+		throw std::runtime_error("Invalid state data");
+	updateSwitchValues();
 }
 
-void A2600System::loadState(EmuApp &, IG::CStringView path)
+size_t A2600System::writeState(std::span<uint8_t> buff, SaveStateFlags flags)
 {
-	Serializer state{path.data(), Serializer::Mode::ReadOnly};
-	if(!osystem.state().loadState(state))
-	{
-		throwFileReadError();
-	}
-	updateSwitchValues();
+	Serializer state;
+	osystem.state().saveState(state);
+	assert(state.size() == saveStateSize);
+	assert(state.size() <= buff.size());
+	state.getByteArray(buff.data(), buff.size());
+	return saveStateSize;
 }
 
 void EmuApp::onCustomizeNavView(EmuApp::NavView &view)

2600.emu/src/main/MainSystem.hh

@@ -66,6 +66,7 @@ class A2600System final: public EmuSystem
 {
 public:
 	OSystem osystem;
+	size_t saveStateSize{};
 	float configuredInputVideoFrameRate{};
 	Properties defaultGameProps{};
 	bool p1DiffB = true, p2DiffB = true, vcsColor = true;
@@ -101,8 +102,9 @@ public:
 	[[gnu::hot]] void runFrame(EmuSystemTaskContext task, EmuVideo *video, EmuAudio *audio);
 	FS::FileString stateFilename(int slot, std::string_view name) const;
 	std::string_view stateFilenameExt() const { return ".sta"; }
-	void loadState(EmuApp &, CStringView uri);
-	void saveState(CStringView path);
+	size_t stateSize() { return saveStateSize; }
+	void readState(EmuApp &, std::span<uint8_t> buff);
+	size_t writeState(std::span<uint8_t> buff, SaveStateFlags = {});
 	bool readConfig(ConfigType, MapIO &io, unsigned key, size_t readSize);
 	void writeConfig(ConfigType, FileIO &);
 	void reset(EmuApp &, ResetMode mode);

C64.emu/src/main/EmuMenuViews.cc

@@ -324,24 +324,21 @@ class CustomFilePathOptionView : public FilePathOptionView, public MainAppHelper
 				app().validSearchPath(system().sysFilePath[0]),
 				[this](CStringView path, FS::file_type type)
 				{
-					if(type == FS::file_type::directory && !appContext().fileUriExists(FS::uriString(path, "DRIVES")))
+					const auto &sysFilePath = system().sysFilePath;
+					if(type == FS::file_type::none && sysFilePath.size() > 1)
 					{
-						app().postErrorMessage("Path is missing DRIVES folder");
-						return false;
+						system().setSystemFilesPath(appContext(), path, type);
+						app().postMessage(5, false, std::format("Using fallback paths:\n{}\n{}", sysFilePath[1], sysFilePath[2]));
 					}
-					logMsg("set firmware path:%s", path.data());
-					systemFilePath.compile(sysPathMenuEntryStr(path), renderer());
-					auto &sysFilePath = system().sysFilePath;
-					sysFilePath[0] = path;
-					if(type == FS::file_type::none)
+					else
 					{
-						if constexpr(Config::envIsLinux)
-							app().postMessage(5, false, std::format("Using fallback paths:\n{}\n{}", sysFilePath[3], sysFilePath[4]));
-						else
+						if(!system().setSystemFilesPath(appContext(), path, type))
 						{
-							app().postMessage(5, false, std::format("Using fallback paths:\n{}\n{}", sysFilePath[1], sysFilePath[2]));
+							app().postErrorMessage("Path is missing DRIVES folder");
+							return false;
 						}
 					}
+					systemFilePath.compile(sysPathMenuEntryStr(path), renderer());
 					return true;
 				});
 			view->appendItem(downloadSystemFiles);

C64.emu/src/main/Main.cc

@@ -20,6 +20,7 @@
 #include <imagine/gui/AlertView.hh>
 #include <imagine/util/format.hh>
 #include <imagine/util/string.h>
+#include <imagine/util/span.hh>
 #include <sys/time.h>
 
 extern "C"
@@ -65,6 +66,7 @@ extern "C"
 namespace EmuEx
 {
 
+constexpr SystemLogger log{"main"};
 const char *EmuSystem::creditsViewStr = CREDITS_INFO_STRING "(c) 2013-2023\nRobert Broglia\nwww.explusalpha.com\n\nPortions (c) the\nVice Team\nvice-emu.sourceforge.io";
 bool EmuSystem::hasPALVideoSystem = true;
 bool EmuSystem::hasResetModes = true;
@@ -243,54 +245,86 @@ FS::FileString C64System::stateFilename(int slot, std::string_view name) const
 
 struct SnapshotTrapData
 {
-	VicePlugin &plugin;
-	const char *pathStr{};
-	bool hasError = true;
+	const VicePlugin &plugin;
+	uint8_t *buffData{};
+	size_t buffSize{};
+	bool hasError{true};
 };
 
+static std::array<char, 32> snapshotVPath(SnapshotTrapData &data)
+{
+	std::array<char, 32> fileStr;
+	// Encode the data/size pointers after the string null terminator
+	if(data.buffData)
+	{
+		auto it = std::ranges::copy(":::B", std::begin(fileStr)).out;
+		it = std::ranges::copy(addressAsBytes(data.buffSize), it).out;
+		std::ranges::copy(asBytes(data.buffData), it);
+		//log.info("encoded size ptr:{} data ptr:{}", (void*)&data.buffSize, (void*)data.buffData);
+	}
+	else
+	{
+		auto it = std::ranges::copy(":::N", std::begin(fileStr)).out;
+		std::ranges::copy(addressAsBytes(data.buffSize), it);
+		//log.info("encoded size ptr:{}", (void*)&data.buffSize);
+	}
+	return fileStr;
+}
+
 static void loadSnapshotTrap(uint16_t, void *data)
 {
-	auto snapData = (SnapshotTrapData*)data;
-	logMsg("loading state: %s", snapData->pathStr);
-	if(snapData->plugin.machine_read_snapshot(snapData->pathStr, 0) < 0)
-		snapData->hasError = true;
+	auto &snapData = *((SnapshotTrapData*)data);
+	assumeExpr(snapData.buffData);
+	log.info("loading state at:{} size:{}", (void*)snapData.buffData, snapData.buffSize);
+	if(snapData.plugin.machine_read_snapshot(snapshotVPath(snapData).data(), 0) < 0)
+		snapData.hasError = true;
 	else
-		snapData->hasError = false;
+		snapData.hasError = false;
 }
 
 static void saveSnapshotTrap(uint16_t, void *data)
 {
-	auto snapData = (SnapshotTrapData*)data;
-	logMsg("saving state: %s", snapData->pathStr);
-	if(snapData->plugin.machine_write_snapshot(snapData->pathStr, 1, 1, 0) < 0)
-		snapData->hasError = true;
+	auto &snapData = *((SnapshotTrapData*)data);
+	log.info("saving state at:{} size:{}", (void*)snapData.buffData, snapData.buffSize);
+	if(snapData.plugin.machine_write_snapshot(snapshotVPath(snapData).data(), 1, 1, 0) < 0)
+		snapData.hasError = true;
 	else
-		snapData->hasError = false;
+		snapData.hasError = false;
 }
 
-void C64System::saveState(IG::CStringView path)
+size_t C64System::stateSize()
 {
-	SnapshotTrapData data{.plugin{plugin}, .pathStr{path}};
+	SnapshotTrapData data{.plugin{plugin}};
 	plugin.interrupt_maincpu_trigger_trap(saveSnapshotTrap, (void*)&data);
 	execC64Frame(); // execute cpu trap
 	if(data.hasError)
-		throwFileWriteError();
+		return 0;
+	return data.buffSize;
 }
 
-void C64System::loadState(EmuApp &, IG::CStringView path)
+void C64System::readState(EmuApp &app, std::span<uint8_t> buff)
 {
 	plugin.vsync_set_warp_mode(0);
-	SnapshotTrapData data{.plugin{plugin}, .pathStr{path}};
+	SnapshotTrapData data{.plugin{plugin}, .buffData = buff.data(), .buffSize = buff.size()};
 	execC64Frame(); // run extra frame in case C64 was just started
 	plugin.interrupt_maincpu_trigger_trap(loadSnapshotTrap, (void*)&data);
 	execC64Frame(); // execute cpu trap, snapshot load may cause reboot from a C64 model change
 	if(data.hasError)
-		return throwFileReadError();
+		throw std::runtime_error("Invalid state data");
 	// reload snapshot in case last load caused a reboot
 	plugin.interrupt_maincpu_trigger_trap(loadSnapshotTrap, (void*)&data);
 	execC64Frame(); // execute cpu trap
 	if(data.hasError)
-		return throwFileReadError();
+		throw std::runtime_error("Invalid state data");
+}
+
+size_t C64System::writeState(std::span<uint8_t> buff, SaveStateFlags flags)
+{
+	SnapshotTrapData data{.plugin{plugin}, .buffData = buff.data(), .buffSize = buff.size()};
+	plugin.interrupt_maincpu_trigger_trap(saveSnapshotTrap, (void*)&data);
+	execC64Frame(); // execute cpu trap
+	assert(!data.hasError);
+	return data.buffSize;
 }
 
 VideoSystem C64System::videoSystem() const

C64.emu/src/main/MainSystem.hh

@@ -20,6 +20,7 @@
 #include <imagine/pixmap/Pixmap.hh>
 #include <imagine/thread/Thread.hh>
 #include <imagine/fs/FS.hh>
+#include <imagine/fs/ArchiveFS.hh>
 #include <emuframework/Option.hh>
 #include <emuframework/EmuSystem.hh>
 #include <vector>
@@ -91,6 +92,7 @@ public:
 	struct video_canvas_s *activeCanvas{};
 	const char *sysFileDir{};
 	VicePlugin plugin{};
+	mutable FS::ArchiveIterator viceSysFilesArchiveIt;
 	std::string defaultPaletteName{};
 	std::string lastMissingSysFile;
 	IG::PixmapView canvasSrcPix{};
@@ -182,14 +184,18 @@ public:
 	bool currSystemIsC64Or128() const;
 	void setRuntimeReuSize(int size);
 	void resetCanvasSourcePixmap(struct video_canvas_s *c);
+	FS::ArchiveIterator &systemFilesArchiveIterator(ApplicationContext, std::string_view path) const;
+	void returnSystemFilesArchiveIO(ArchiveIO);
+	bool setSystemFilesPath(ApplicationContext, CStringView path, FS::file_type);
 
 	// required API functions
 	void loadContent(IO &, EmuSystemCreateParams, OnLoadProgressDelegate);
 	[[gnu::hot]] void runFrame(EmuSystemTaskContext task, EmuVideo *video, EmuAudio *audio);
 	FS::FileString stateFilename(int slot, std::string_view name) const;
 	std::string_view stateFilenameExt() const { return ".vsf"; }
-	void loadState(EmuApp &, CStringView uri);
-	void saveState(CStringView path);
+	size_t stateSize();
+	void readState(EmuApp &, std::span<uint8_t> buff);
+	size_t writeState(std::span<uint8_t> buff, SaveStateFlags = {});
 	bool readConfig(ConfigType, MapIO &io, unsigned key, size_t readSize);
 	void writeConfig(ConfigType, FileIO &);
 	void reset(EmuApp &, ResetMode mode);

C64.emu/src/main/VicePlugin.cc

@@ -277,14 +277,14 @@ int VicePlugin::resources_get_default_value(const char *name, void *value_return
 	return -1;
 }
 
-int VicePlugin::machine_write_snapshot(const char *name, int save_roms, int save_disks, int even_mode)
+int VicePlugin::machine_write_snapshot(const char *name, int save_roms, int save_disks, int even_mode) const
 {
 	if(machine_write_snapshot_)
 		return machine_write_snapshot_(name, save_roms, save_disks, even_mode);
 	return -1;
 }
 
-int VicePlugin::machine_read_snapshot(const char *name, int event_mode)
+int VicePlugin::machine_read_snapshot(const char *name, int event_mode) const
 {
 	if(machine_read_snapshot_)
 		return machine_read_snapshot_(name, event_mode);
@@ -308,7 +308,7 @@ struct drive_type_info_s *VicePlugin::machine_drive_get_type_info_list()
 	return machine_drive_get_type_info_list_();
 }
 
-void VicePlugin::interrupt_maincpu_trigger_trap(void trap_func(uint16_t, void *data), void *data)
+void VicePlugin::interrupt_maincpu_trigger_trap(void trap_func(uint16_t, void *data), void *data) const
 {
 	if(interrupt_maincpu_trigger_trap_)
 		interrupt_maincpu_trigger_trap_(trap_func, data);

C64.emu/src/main/VicePlugin.hh

@@ -111,12 +111,12 @@ struct VicePlugin
 	int resources_get_int(const char *name, int *value_return) const;
 	int resources_set_int(const char *name, int value);
 	int resources_get_default_value(const char *name, void *value_return) const;
-	int machine_write_snapshot(const char *name, int save_roms, int save_disks, int even_mode);
-	int machine_read_snapshot(const char *name, int event_mode);
+	int machine_write_snapshot(const char *name, int save_roms, int save_disks, int even_mode) const;
+	int machine_read_snapshot(const char *name, int event_mode) const;
 	void machine_set_restore_key(int v);
 	void machine_trigger_reset(const unsigned int mode);
 	struct drive_type_info_s *machine_drive_get_type_info_list();
-	void interrupt_maincpu_trigger_trap(void trap_func(uint16_t, void *data), void *data);
+	void interrupt_maincpu_trigger_trap(void trap_func(uint16_t, void *data), void *data) const;
 	int init_main();
 	void maincpu_mainloop();
 	int autostart_autodetect(const char *file_name, const char *program_name,