Fix minetest.dig_node returning true when node isn't diggable (#10890)

Author: rubenwardy

builtin/game/item.lua

@@ -557,7 +557,7 @@ function core.node_dig(pos, node, digger)
 		log("info", diggername .. " tried to dig "
 			.. node.name .. " which is not diggable "
 			.. core.pos_to_string(pos))
-		return
+		return false
 	end
 
 	if core.is_protected(pos, diggername) then
@@ -566,7 +566,7 @@ function core.node_dig(pos, node, digger)
 				.. " at protected position "
 				.. core.pos_to_string(pos))
 		core.record_protection_violation(pos, diggername)
-		return
+		return false
 	end
 
 	log('action', diggername .. " digs "
@@ -649,6 +649,8 @@ function core.node_dig(pos, node, digger)
 		local node_copy = {name=node.name, param1=node.param1, param2=node.param2}
 		callback(pos_copy, node_copy, digger)
 	end
+
+	return true
 end
 
 function core.itemstring_with_palette(item, palette_index)

doc/lua_api.txt

@@ -7628,6 +7628,8 @@ Used by `minetest.register_node`.
         on_dig = function(pos, node, digger),
         -- default: minetest.node_dig
         -- By default checks privileges, wears out tool and removes node.
+        -- return true if the node was dug successfully, false otherwise.
+        -- Deprecated: returning nil is the same as returning true.
 
         on_timer = function(pos, elapsed),
         -- default: nil

src/script/cpp_api/s_node.cpp

@@ -141,9 +141,14 @@ bool ScriptApiNode::node_on_dig(v3s16 p, MapNode node,
 	push_v3s16(L, p);
 	pushnode(L, node, ndef);
 	objectrefGetOrCreate(L, digger);
-	PCALL_RES(lua_pcall(L, 3, 0, error_handler));
-	lua_pop(L, 1);  // Pop error handler
-	return true;
+	PCALL_RES(lua_pcall(L, 3, 1, error_handler));
+
+	// nil is treated as true for backwards compat
+	bool result = lua_isnil(L, -1) || lua_toboolean(L, -1);
+
+	lua_pop(L, 2);  // Pop error handler and result
+
+	return result;
 }
 
 void ScriptApiNode::node_on_construct(v3s16 p, MapNode node)