[no squash] Bypass media transfer in single player

.github/workflows/linux.yml

@@ -86,23 +86,20 @@ jobs:
       - name: Install deps
         run: |
           source ./util/ci/common.sh
-          install_linux_deps clang-7 valgrind
+          install_linux_deps clang-7 llvm
 
       - name: Build
         run: |
           ./util/ci/build.sh
         env:
           CC: clang-7
           CXX: clang++-7
+          CMAKE_FLAGS: '-DCMAKE_C_FLAGS="-fsanitize=address" -DCMAKE_CXX_FLAGS="-fsanitize=address"'
 
       - name: Unittest
         run: |
           ./bin/minetest --run-unittests
 
-      - name: Valgrind
-        run: |
-          valgrind --leak-check=full --leak-check-heuristics=all --undef-value-errors=no --error-exitcode=9 ./bin/minetest --run-unittests
-
   # Current clang version
   clang_14:
     runs-on: ubuntu-22.04

src/client/clientmedia.cpp

@@ -29,6 +29,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "util/serialize.h"
 #include "util/sha1.h"
 #include "util/string.h"
+#include <sstream>
 
 static std::string getMediaCacheDir()
 {
@@ -41,7 +42,16 @@ bool clientMediaUpdateCache(const std::string &raw_hash, const std::string &file
 	std::string sha1_hex = hex_encode(raw_hash);
 	if (!media_cache.exists(sha1_hex))
 		return media_cache.update(sha1_hex, filedata);
-	return true;
+	return false;
+}
+
+bool clientMediaUpdateCacheCopy(const std::string &raw_hash, const std::string &path)
+{
+	FileCache media_cache(getMediaCacheDir());
+	std::string sha1_hex = hex_encode(raw_hash);
+	if (!media_cache.exists(sha1_hex))
+		return media_cache.updateCopyFile(sha1_hex, path);
+	return false;
 }
 
 /*
@@ -189,10 +199,6 @@ void ClientMediaDownloader::initialStep(Client *client)
 
 	assert(m_uncached_received_count == 0);
 
-	// Create the media cache dir if we are likely to write to it
-	if (m_uncached_count != 0)
-		createCacheDirs();
-
 	// If we found all files in the cache, report this fact to the server.
 	// If the server reported no remote servers, immediately start
 	// conventional transfers. Note: if cURL support is not compiled in,
@@ -511,18 +517,6 @@ IClientMediaDownloader::IClientMediaDownloader():
 {
 }
 
-void IClientMediaDownloader::createCacheDirs()
-{
-	if (!m_write_to_cache)
-		return;
-
-	std::string path = getMediaCacheDir();
-	if (!fs::CreateAllDirs(path)) {
-		errorstream << "Client: Could not create media cache directory: "
-			<< path << std::endl;
-	}
-}
-
 bool IClientMediaDownloader::tryLoadFromCache(const std::string &name,
 	const std::string &sha1, Client *client)
 {
@@ -726,8 +720,6 @@ void SingleMediaDownloader::initialStep(Client *client)
 	if (isDone())
 		return;
 
-	createCacheDirs();
-
 	// If the server reported no remote servers, immediately fall back to
 	// conventional transfer.
 	if (!USE_CURL || m_remotes.empty()) {

src/client/clientmedia.h

@@ -22,7 +22,6 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "irrlichttypes.h"
 #include "filecache.h"
 #include "util/basic_macros.h"
-#include <ostream>
 #include <map>
 #include <set>
 #include <vector>
@@ -35,10 +34,15 @@ struct HTTPFetchResult;
 #define MTHASHSET_FILE_NAME "index.mth"
 
 // Store file into media cache (unless it exists already)
-// Validating the hash is responsibility of the caller
+// Caller should check the hash.
+// return true if something was updated
 bool clientMediaUpdateCache(const std::string &raw_hash,
 	const std::string &filedata);
 
+// Copy file on disk(!) into media cache (unless it exists already)
+bool clientMediaUpdateCacheCopy(const std::string &raw_hash,
+	const std::string &path);
+
 // more of a base class than an interface but this name was most convenient...
 class IClientMediaDownloader
 {
@@ -81,8 +85,6 @@ class IClientMediaDownloader
 	virtual bool loadMedia(Client *client, const std::string &data,
 		const std::string &name) = 0;
 
-	void createCacheDirs();
-
 	bool tryLoadFromCache(const std::string &name, const std::string &sha1,
 			Client *client);
 

src/client/filecache.cpp

@@ -28,6 +28,14 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include <fstream>
 #include <cstdlib>
 
+void FileCache::createDir()
+{
+	if (!fs::CreateAllDirs(m_dir)) {
+		errorstream << "Could not create cache directory: "
+			<< m_dir << std::endl;
+	}
+}
+
 bool FileCache::loadByPath(const std::string &path, std::ostream &os)
 {
 	std::ifstream fis(path.c_str(), std::ios_base::binary);
@@ -40,8 +48,8 @@ bool FileCache::loadByPath(const std::string &path, std::ostream &os)
 
 	bool bad = false;
 	for(;;){
-		char buf[1024];
-		fis.read(buf, 1024);
+		char buf[4096];
+		fis.read(buf, sizeof(buf));
 		std::streamsize len = fis.gcount();
 		os.write(buf, len);
 		if(fis.eof())
@@ -61,6 +69,7 @@ bool FileCache::loadByPath(const std::string &path, std::ostream &os)
 
 bool FileCache::updateByPath(const std::string &path, const std::string &data)
 {
+	createDir();
 	std::ofstream file(path.c_str(), std::ios_base::binary |
 			std::ios_base::trunc);
 
@@ -95,3 +104,11 @@ bool FileCache::exists(const std::string &name)
 	std::ifstream fis(path.c_str(), std::ios_base::binary);
 	return fis.good();
 }
+
+bool FileCache::updateCopyFile(const std::string &name, const std::string &src_path)
+{
+	std::string path = m_dir + DIR_DELIM + name;
+
+	createDir();
+	return fs::CopyFileContents(src_path, path);
+}

src/client/filecache.h

@@ -35,9 +35,13 @@ class FileCache
 	bool load(const std::string &name, std::ostream &os);
 	bool exists(const std::string &name);
 
+	// Copy another file on disk into the cache
+	bool updateCopyFile(const std::string &name, const std::string &src_path);
+
 private:
 	std::string m_dir;
 
+	void createDir();
 	bool loadByPath(const std::string &path, std::ostream &os);
 	bool updateByPath(const std::string &path, const std::string &data);
 };

src/client/game.cpp

@@ -33,6 +33,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "client/mapblock_mesh.h"
 #include "client/sound.h"
 #include "clientmap.h"
+#include "clientmedia.h" // For clientMediaUpdateCacheCopy
 #include "clouds.h"
 #include "config.h"
 #include "content_cao.h"
@@ -776,6 +777,7 @@ class Game {
 	bool initSound();
 	bool createSingleplayerServer(const std::string &map_dir,
 			const SubgameSpec &gamespec, u16 port);
+	void copyServerClientCache();
 
 	// Client creation
 	bool createClient(const GameStartData &start_data);
@@ -1459,9 +1461,31 @@ bool Game::createSingleplayerServer(const std::string &map_dir,
 			false, nullptr, error_message);
 	server->start();
 
+	copyServerClientCache();
+
 	return true;
 }
 
+void Game::copyServerClientCache()
+{
+	// It would be possible to let the client directly read the media files
+	// from where the server knows they are. But aside from being more complicated
+	// it would also *not* fill the media cache and cause slower joining of 
+	// remote servers.
+	// (Imagine that you launch a game once locally and then connect to a server.)
+
+	assert(server);
+	auto map = server->getMediaList();
+	u32 n = 0;
+	for (auto &it : map) {
+		assert(it.first.size() == 20); // SHA1
+		if (clientMediaUpdateCacheCopy(it.first, it.second))
+			n++;
+	}
+	infostream << "Copied " << n << " files directly from server to client cache"
+		<< std::endl;
+}
+
 bool Game::createClient(const GameStartData &start_data)
 {
 	showOverlayMessage(N_("Creating client..."), 0, 10);