Send only relevant ItemStack meta fields to Client

builtin/settingtypes.txt

@@ -1062,6 +1062,11 @@ max_packets_per_iteration (Max. packets per iteration) int 1024
 #     (levels 1-3 use Zlib's "fast" method, 4-9 use the normal method)
 map_compression_level_net (Map Compression Level for Network Transfer) int -1 -1 9
 
+#    Sends all item metadata fields to clients. Otherwise the server will truncate
+#    irrelevant data per-client to reduce load and network traffic.
+#    Note: Set to "true" to allow local map saving on clients prior to 5.5.0-dev.
+send_all_item_metadata (Send all Item metadata to clients) bool false
+
 [*Game]
 
 #    Default game when creating a new world.

doc/lua_api.txt

@@ -2040,6 +2040,12 @@ Some of the values in the key-value store are handled specially:
 * `palette_index`: If the item has a palette, this is used to get the
   current color from the palette.
 
+Item metadata is usually not fully sent to client (except the client has
+local map saving enabled or it is disabled globaly by the server). Only
+the fields `description`, `short_description`, `color`, `palette_index`
+and `tool_capabilities` are always sent. If there are any other fields,
+they will be hashed into an additional `_hash` field.
+
 Example:
 
     local meta = stack:get_meta()

minetest.conf.example

@@ -1269,6 +1269,11 @@
 #    type: int min: -1 max: 9
 # map_compression_level_net = -1
 
+#    Sends all item metadata fields to clients. Otherwise the server will truncate
+#    irrelevant data per-client to reduce load and network traffic.
+#    Note: Set to "true" to allow local map saving on clients prior to 5.5.0-dev.
+# send_all_item_metadata = false
+
 ## Game
 
 #    Default game when creating a new world.

src/client/client.cpp

@@ -1008,14 +1008,15 @@ AuthMechanism Client::choseAuthMech(const u32 mechs)
 
 void Client::sendInit(const std::string &playerName)
 {
-	NetworkPacket pkt(TOSERVER_INIT, 1 + 2 + 2 + (1 + playerName.size()));
+	NetworkPacket pkt(TOSERVER_INIT, 1 + 2 + 2 + (1 + playerName.size()) + 1);
 
 	// we don't support network compression yet
 	u16 supp_comp_modes = NETPROTO_COMPRESSION_NONE;
 
 	pkt << (u8) SER_FMT_VER_HIGHEST_READ << (u16) supp_comp_modes;
 	pkt << (u16) CLIENT_PROTOCOL_VERSION_MIN << (u16) CLIENT_PROTOCOL_VERSION_MAX;
 	pkt << playerName;
+	pkt << (u8) (m_localdb ? 1 : 0);
 
 	Send(&pkt);
 }

src/clientiface.h

@@ -233,6 +233,8 @@ class RemoteClient
 	//
 	u16 net_proto_version = 0;
 
+	u8 mapsaving_enabled = 0;
+
 	/* Authentication information */
 	std::string enc_pwd = "";
 	bool create_player_on_auth_success = false;

src/defaultsettings.cpp

@@ -391,6 +391,7 @@ void set_default_settings()
 	settings->setDefault("sqlite_synchronous", "2");
 	settings->setDefault("map_compression_level_disk", "3");
 	settings->setDefault("map_compression_level_net", "-1");
+	settings->setDefault("send_all_item_metadata", "false");
 	settings->setDefault("full_block_send_enable_min_time_from_building", "2.0");
 	settings->setDefault("dedicated_server_step", "0.09");
 	settings->setDefault("active_block_mgmt_interval", "2.0");

src/inventory.cpp

@@ -56,7 +56,7 @@ ItemStack::ItemStack(const std::string &name_, u16 count_,
 		count = 1;
 }
 
-void ItemStack::serialize(std::ostream &os, bool serialize_meta) const
+void ItemStack::serialize(std::ostream &os, InventoryOptimizationOption opt) const
 {
 	if (empty())
 		return;
@@ -77,10 +77,10 @@ void ItemStack::serialize(std::ostream &os, bool serialize_meta) const
 		os << " " << wear;
 	if (parts >= 4) {
 		os << " ";
-		if (serialize_meta)
-			metadata.serialize(os);
-		else
+		if (opt & INV_OO_NO_META)
 			os << "<metadata size=" << metadata.size() << ">";
+		else
+			metadata.serialize(os, opt);
 	}
 }
 
@@ -243,10 +243,10 @@ void ItemStack::deSerialize(const std::string &str, IItemDefManager *itemdef)
 	deSerialize(is, itemdef);
 }
 
-std::string ItemStack::getItemString(bool include_meta) const
+std::string ItemStack::getItemString(InventoryOptimizationOption opt) const
 {
 	std::ostringstream os(std::ios::binary);
-	serialize(os, include_meta);
+	serialize(os, opt);
 	return os.str();
 }
 
@@ -425,7 +425,7 @@ void InventoryList::setName(const std::string &name)
 	setModified();
 }
 
-void InventoryList::serialize(std::ostream &os, bool incremental) const
+void InventoryList::serialize(std::ostream &os, InventoryOptimizationOption opt) const
 {
 	//os.imbue(std::locale("C"));
 
@@ -436,10 +436,10 @@ void InventoryList::serialize(std::ostream &os, bool incremental) const
 			os<<"Empty";
 		} else {
 			os<<"Item ";
-			item.serialize(os);
+			item.serialize(os, opt);
 		}
 		// TODO: Implement this:
-		// if (!incremental || item.checkModified())
+		// if (opt & INV_OO_INCREMENTAL && !item.checkModified())
 		// os << "Keep";
 		os<<"\n";
 	}
@@ -847,15 +847,15 @@ bool Inventory::operator == (const Inventory &other) const
 	return true;
 }
 
-void Inventory::serialize(std::ostream &os, bool incremental) const
+void Inventory::serialize(std::ostream &os, InventoryOptimizationOption opt) const
 {
 	//std::cout << "Serialize " << (int)incremental << ", n=" << m_lists.size() << std::endl;
 	for (const InventoryList *list : m_lists) {
-		if (!incremental || list->checkModified()) {
-			os << "List " << list->getName() << " " << list->getSize() << "\n";
-			list->serialize(os, incremental);
-		} else {
+		if (opt & INV_OO_INCREMENTAL && !list->checkModified()) {
 			os << "KeepList " << list->getName() << "\n";
+		} else {
+			os << "List " << list->getName() << " " << list->getSize() << "\n";
+			list->serialize(os, opt);
 		}
 	}
 

src/inventory.h

@@ -19,6 +19,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 
 #pragma once
 
+#include "inventoryoptimisation.h"
 #include "itemdef.h"
 #include "irrlichttypes.h"
 #include "itemstackmetadata.h"
@@ -40,13 +41,13 @@ struct ItemStack
 	~ItemStack() = default;
 
 	// Serialization
-	void serialize(std::ostream &os, bool serialize_meta = true) const;
+	void serialize(std::ostream &os, InventoryOptimizationOption opt = INV_OO_NONE) const;
 	// Deserialization. Pass itemdef unless you don't want aliases resolved.
 	void deSerialize(std::istream &is, IItemDefManager *itemdef = NULL);
 	void deSerialize(const std::string &s, IItemDefManager *itemdef = NULL);
 
 	// Returns the string used for inventory
-	std::string getItemString(bool include_meta = true) const;
+	std::string getItemString(InventoryOptimizationOption opt = INV_OO_NONE) const;
 	// Returns the tooltip
 	std::string getDescription(IItemDefManager *itemdef) const;
 	std::string getShortDescription(IItemDefManager *itemdef) const;
@@ -195,7 +196,7 @@ class InventoryList
 	void setSize(u32 newsize);
 	void setWidth(u32 newWidth);
 	void setName(const std::string &name);
-	void serialize(std::ostream &os, bool incremental) const;
+	void serialize(std::ostream &os, InventoryOptimizationOption opt = INV_OO_NONE) const;
 	void deSerialize(std::istream &is);
 
 	InventoryList(const InventoryList &other);
@@ -295,7 +296,7 @@ class Inventory
 	}
 
 	// Never ever serialize to disk using "incremental"!
-	void serialize(std::ostream &os, bool incremental = false) const;
+	void serialize(std::ostream &os, InventoryOptimizationOption opt = INV_OO_NONE) const;
 	void deSerialize(std::istream &is);
 
 	InventoryList * addList(const std::string &name, u32 size);

src/inventoryoptimisation.h

@@ -0,0 +1,28 @@
+/*
+Minetest
+Copyright (C) 2021 Elias Fleckenstein <eliasfleckenstein@web.de>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation; either version 2.1 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+*/
+
+#pragma once
+
+enum InventoryOptimizationOption {
+	INV_OO_NONE = 0,
+	INV_OO_INCREMENTAL = 0x0001,
+	INV_OO_META_SPARSE = 0x0002,
+	INV_OO_INCREMENTAL_META_SPARSE = 0x0003,
+	INV_OO_NO_META  = 0x0004,
+};

src/itemstackmetadata.cpp

@@ -21,6 +21,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "itemstackmetadata.h"
 #include "util/serialize.h"
 #include "util/strfnd.h"
+#include "util/numeric.h"
 #include <algorithm>
 
 #define DESERIALIZE_START '\x01'
@@ -58,15 +59,28 @@ bool ItemStackMetadata::setString(const std::string &name, const std::string &va
 	return result;
 }
 
-void ItemStackMetadata::serialize(std::ostream &os) const
+void ItemStackMetadata::serialize(std::ostream &os, InventoryOptimizationOption opt) const
 {
 	std::ostringstream os2;
+	std::ostringstream os_hash;
 	os2 << DESERIALIZE_START;
+	bool sparse_meta = opt & INV_OO_META_SPARSE;
 	for (const auto &stringvar : m_stringvars) {
+		bool silent = sparse_meta
+			&& stringvar.first != TOOLCAP_KEY
+			&& stringvar.first != "description"
+			&& stringvar.first != "color"
+			&& stringvar.first != "short_description"
+			&& stringvar.first != "palette_index";
 		if (!stringvar.first.empty() || !stringvar.second.empty())
-			os2 << stringvar.first << DESERIALIZE_KV_DELIM
+			(silent ? os_hash : os2) << stringvar.first << DESERIALIZE_KV_DELIM
 				<< stringvar.second << DESERIALIZE_PAIR_DELIM;
 	}
+	std::string hash_str = os_hash.str();
+	if (! hash_str.empty()) {
+		os2 << "_hash" << DESERIALIZE_KV_DELIM
+			<< murmur_hash_64_ua(hash_str.data(), hash_str.length(), 0xdeadbeef) << DESERIALIZE_PAIR_DELIM;
+	}
 	os << serializeJsonStringIfNeeded(os2.str());
 }
 

src/itemstackmetadata.h

@@ -19,6 +19,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 
 #pragma once
 
+#include "inventoryoptimisation.h"
 #include "metadata.h"
 #include "tool.h"
 
@@ -34,7 +35,7 @@ class ItemStackMetadata : public Metadata
 	void clear() override;
 	bool setString(const std::string &name, const std::string &var) override;
 
-	void serialize(std::ostream &os) const;
+	void serialize(std::ostream &os, InventoryOptimizationOption opt = INV_OO_NONE) const;
 	void deSerialize(std::istream &is);
 
 	const ToolCapabilities &getToolCapabilities(

src/mapblock.cpp

@@ -355,7 +355,7 @@ static void correctBlockNodeIds(const NameIdMapping *nimap, MapNode *nodes,
 	}
 }
 
-void MapBlock::serialize(std::ostream &os, u8 version, bool disk, int compression_level)
+void MapBlock::serialize(std::ostream &os, u8 version, bool disk, int compression_level, InventoryOptimizationOption opt)
 {
 	if(!ser_ver_supported(version))
 		throw VersionMismatchException("ERROR: MapBlock format not supported");
@@ -411,7 +411,7 @@ void MapBlock::serialize(std::ostream &os, u8 version, bool disk, int compressio
 		Node metadata
 	*/
 	std::ostringstream oss(std::ios_base::binary);
-	m_node_metadata.serialize(oss, version, disk);
+	m_node_metadata.serialize(oss, version, disk, false, opt);
 	compressZlib(oss.str(), os, compression_level);
 
 	/*

src/mapblock.h

@@ -21,6 +21,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 
 #include <set>
 #include "irr_v3d.h"
+#include "inventoryoptimisation.h"
 #include "mapnode.h"
 #include "exceptions.h"
 #include "constants.h"
@@ -473,7 +474,7 @@ class MapBlock
 	// These don't write or read version by itself
 	// Set disk to true for on-disk format, false for over-the-network format
 	// Precondition: version >= SER_FMT_VER_LOWEST_WRITE
-	void serialize(std::ostream &os, u8 version, bool disk, int compression_level);
+	void serialize(std::ostream &os, u8 version, bool disk, int compression_level, InventoryOptimizationOption opt = INV_OO_NONE);
 	// If disk == true: In addition to doing other things, will add
 	// unknown blocks from id-name mapping to wndef
 	void deSerialize(std::istream &is, u8 version, bool disk);

src/network/serverpackethandler.cpp

@@ -100,10 +100,17 @@ void Server::handleCommand_Init(NetworkPacket* pkt)
 	u16 min_net_proto_version = 0;
 	u16 max_net_proto_version;
 	std::string playerName;
+	u8 mapsaving_enabled = 0;
 
 	*pkt >> client_max >> supp_compr_modes >> min_net_proto_version
 			>> max_net_proto_version >> playerName;
 
+	try {
+		*pkt >> mapsaving_enabled;
+	} catch (PacketError &e) {};
+
+	client->mapsaving_enabled = mapsaving_enabled;
+
 	u8 our_max = SER_FMT_VER_HIGHEST_READ;
 	// Use the highest version supported by both
 	u8 depl_serial_v = std::min(client_max, our_max);

src/nodemetadata.cpp

@@ -40,7 +40,7 @@ NodeMetadata::~NodeMetadata()
 	delete m_inventory;
 }
 
-void NodeMetadata::serialize(std::ostream &os, u8 version, bool disk) const
+void NodeMetadata::serialize(std::ostream &os, u8 version, bool disk, InventoryOptimizationOption opt) const
 {
 	int num_vars = disk ? m_stringvars.size() : countNonPrivate();
 	writeU32(os, num_vars);
@@ -55,7 +55,7 @@ void NodeMetadata::serialize(std::ostream &os, u8 version, bool disk) const
 			writeU8(os, (priv) ? 1 : 0);
 	}
 
-	m_inventory->serialize(os);
+	m_inventory->serialize(os, opt);
 }
 
 void NodeMetadata::deSerialize(std::istream &is, u8 version)
@@ -113,7 +113,7 @@ int NodeMetadata::countNonPrivate() const
 */
 
 void NodeMetadataList::serialize(std::ostream &os, u8 blockver, bool disk,
-	bool absolute_pos) const
+	bool absolute_pos, InventoryOptimizationOption opt) const
 {
 	/*
 		Version 0 is a placeholder for "nothing to see here; go away."
@@ -146,7 +146,7 @@ void NodeMetadataList::serialize(std::ostream &os, u8 blockver, bool disk,
 			u16 p16 = (p.Z * MAP_BLOCKSIZE + p.Y) * MAP_BLOCKSIZE + p.X;
 			writeU16(os, p16);
 		}
-		data->serialize(os, version, disk);
+		data->serialize(os, version, disk, opt);
 	}
 }
 

src/nodemetadata.h

@@ -20,6 +20,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #pragma once
 
 #include <unordered_set>
+#include "inventoryoptimisation.h"
 #include "metadata.h"
 
 /*
@@ -40,7 +41,7 @@ class NodeMetadata : public Metadata
 	NodeMetadata(IItemDefManager *item_def_mgr);
 	~NodeMetadata();
 
-	void serialize(std::ostream &os, u8 version, bool disk=true) const;
+	void serialize(std::ostream &os, u8 version, bool disk = true, InventoryOptimizationOption opt = INV_OO_NONE) const;
 	void deSerialize(std::istream &is, u8 version);
 
 	void clear();
@@ -82,7 +83,7 @@ class NodeMetadataList
 	~NodeMetadataList();
 
 	void serialize(std::ostream &os, u8 blockver, bool disk = true,
-		bool absolute_pos = false) const;
+		bool absolute_pos = false, InventoryOptimizationOption opt = INV_OO_NONE) const;
 	void deSerialize(std::istream &is, IItemDefManager *item_def_mgr,
 		bool absolute_pos = false);