Lua on each mapgen thread (#13092)

.luacheckrc

@@ -17,6 +17,7 @@ read_globals = {
 	"VoxelArea",
 	"profiler",
 	"Settings",
+	"PerlinNoise", "PerlinNoiseMap",
 
 	string = {fields = {"split", "trim"}},
 	table  = {fields = {"copy", "getn", "indexof", "insert_all"}},

builtin/emerge/env.lua

@@ -0,0 +1,61 @@
+-- Reimplementations of some environment function on vmanips, since this is
+-- what the emerge environment operates on
+
+-- core.vmanip = <VoxelManip> -- set by C++
+
+function core.set_node(pos, node)
+	return core.vmanip:set_node_at(pos, node)
+end
+
+function core.bulk_set_node(pos_list, node)
+	local vm = core.vmanip
+	local set_node_at = vm.set_node_at
+	for _, pos in ipairs(pos_list) do
+		if not set_node_at(vm, pos, node) then
+			return false
+		end
+	end
+	return true
+end
+
+core.add_node = core.set_node
+
+-- we don't deal with metadata currently
+core.swap_node = core.set_node
+
+function core.remove_node(pos)
+	return core.vmanip:set_node_at(pos, {name="air"})
+end
+
+function core.get_node(pos)
+	return core.vmanip:get_node_at(pos)
+end
+
+function core.get_node_or_nil(pos)
+	local node = core.vmanip:get_node_at(pos)
+	return node.name ~= "ignore" and node
+end
+
+function core.get_perlin(seed, octaves, persist, spread)
+	local params
+	if type(seed) == "table" then
+		params = table.copy(seed)
+	else
+		assert(type(seed) == "number")
+		params = {
+			seed = seed,
+			octaves = octaves,
+			persist = persist,
+			spread = {x=spread, y=spread, z=spread},
+		}
+	end
+	params.seed = core.get_seed(params.seed) -- add mapgen seed
+	return PerlinNoise(params)
+end
+
+
+function core.get_perlin_map(params, size)
+	local params2 = table.copy(params)
+	params2.seed = core.get_seed(params.seed) -- add mapgen seed
+	return PerlinNoiseMap(params2, size)
+end

builtin/emerge/init.lua

@@ -0,0 +1,21 @@
+local gamepath = core.get_builtin_path() .. "game" .. DIR_DELIM
+local commonpath = core.get_builtin_path() .. "common" .. DIR_DELIM
+local epath = core.get_builtin_path() .. "emerge" .. DIR_DELIM
+
+local builtin_shared = {}
+
+-- Import parts shared with "game" environment
+dofile(gamepath .. "constants.lua")
+assert(loadfile(commonpath .. "item_s.lua"))(builtin_shared)
+dofile(gamepath .. "misc_s.lua")
+dofile(gamepath .. "features.lua")
+dofile(gamepath .. "voxelarea.lua")
+
+-- Now for our own stuff
+assert(loadfile(commonpath .. "register.lua"))(builtin_shared)
+assert(loadfile(epath .. "register.lua"))(builtin_shared)
+dofile(epath .. "env.lua")
+
+builtin_shared.cache_content_ids()
+
+core.log("info", "Initialized emerge Lua environment")

builtin/emerge/register.lua

@@ -0,0 +1,54 @@
+local builtin_shared = ...
+
+-- Copy all the registration tables over
+do
+	local all = assert(core.transferred_globals)
+	core.transferred_globals = nil
+
+	all.registered_nodes = {}
+	all.registered_craftitems = {}
+	all.registered_tools = {}
+	for k, v in pairs(all.registered_items) do
+		-- Disable further modification
+		setmetatable(v, {__newindex = {}})
+		-- Reassemble the other tables
+		if v.type == "node" then
+			getmetatable(v).__index = all.nodedef_default
+			all.registered_nodes[k] = v
+		elseif v.type == "craft" then
+			getmetatable(v).__index = all.craftitemdef_default
+			all.registered_craftitems[k] = v
+		elseif v.type == "tool" then
+			getmetatable(v).__index = all.tooldef_default
+			all.registered_tools[k] = v
+		else
+			getmetatable(v).__index = all.noneitemdef_default
+		end
+	end
+
+	for k, v in pairs(all) do
+		core[k] = v
+	end
+end
+
+-- For tables that are indexed by item name:
+-- If table[X] does not exist, default to table[core.registered_aliases[X]]
+local alias_metatable = {
+	__index = function(t, name)
+		return rawget(t, core.registered_aliases[name])
+	end
+}
+setmetatable(core.registered_items, alias_metatable)
+setmetatable(core.registered_nodes, alias_metatable)
+setmetatable(core.registered_craftitems, alias_metatable)
+setmetatable(core.registered_tools, alias_metatable)
+
+--
+-- Callbacks
+--
+
+local make_registration = builtin_shared.make_registration
+
+core.registered_on_mods_loaded, core.register_on_mods_loaded = make_registration()
+core.registered_on_generateds, core.register_on_generated = make_registration()
+core.registered_on_shutdown, core.register_on_shutdown = make_registration()

builtin/game/misc.lua

@@ -237,8 +237,8 @@ end
 core.dynamic_media_callbacks = {}
 
 
--- Transfer of certain globals into async environment
--- see builtin/async/game.lua for the other side
+-- Transfer of certain globals into seconday Lua environments
+-- see builtin/async/game.lua or builtin/emerge/register.lua for the unpacking
 
 local function copy_filtering(t, seen)
 	if type(t) == "userdata" or type(t) == "function" then
@@ -261,6 +261,9 @@ function core.get_globals_to_transfer()
 	local all = {
 		registered_items = copy_filtering(core.registered_items),
 		registered_aliases = core.registered_aliases,
+		registered_biomes = core.registered_biomes,
+		registered_ores = core.registered_ores,
+		registered_decorations = core.registered_decorations,
 
 		nodedef_default = copy_filtering(core.nodedef_default),
 		craftitemdef_default = copy_filtering(core.craftitemdef_default),

builtin/init.lua

@@ -31,8 +31,6 @@ minetest = core
 
 -- Load other files
 local scriptdir = core.get_builtin_path()
-local gamepath = scriptdir .. "game" .. DIR_DELIM
-local clientpath = scriptdir .. "client" .. DIR_DELIM
 local commonpath = scriptdir .. "common" .. DIR_DELIM
 local asyncpath = scriptdir .. "async" .. DIR_DELIM
 
@@ -42,7 +40,7 @@ dofile(commonpath .. "serialize.lua")
 dofile(commonpath .. "misc_helpers.lua")
 
 if INIT == "game" then
-	dofile(gamepath .. "init.lua")
+	dofile(scriptdir .. "game" .. DIR_DELIM .. "init.lua")
 	assert(not core.get_http_api)
 elseif INIT == "mainmenu" then
 	local mm_script = core.settings:get("main_menu_script")
@@ -67,7 +65,9 @@ elseif INIT == "async"  then
 elseif INIT == "async_game" then
 	dofile(asyncpath .. "game.lua")
 elseif INIT == "client" then
-	dofile(clientpath .. "init.lua")
+	dofile(scriptdir .. "client" .. DIR_DELIM .. "init.lua")
+elseif INIT == "emerge" then
+	dofile(scriptdir .. "emerge" .. DIR_DELIM .. "init.lua")
 else
 	error(("Unrecognized builtin initialization type %s!"):format(tostring(INIT)))
 end

doc/lua_api.md

@@ -4679,6 +4679,7 @@ differences:
   into it; it's not necessary to call `VoxelManip:read_from_map()`.
   Note that the region of map it has loaded is NOT THE SAME as the `minp`, `maxp`
   parameters of `on_generated()`. Refer to `minetest.get_mapgen_object` docs.
+  Once you're done you still need to call `VoxelManip:write_to_map()`
 
 * The `on_generated()` callbacks of some mods may place individual nodes in the
   generated area using non-VoxelManip map modification methods. Because the
@@ -4875,10 +4876,10 @@ Mapgen objects
 ==============
 
 A mapgen object is a construct used in map generation. Mapgen objects can be
-used by an `on_generate` callback to speed up operations by avoiding
+used by an `on_generated` callback to speed up operations by avoiding
 unnecessary recalculations, these can be retrieved using the
 `minetest.get_mapgen_object()` function. If the requested Mapgen object is
-unavailable, or `get_mapgen_object()` was called outside of an `on_generate()`
+unavailable, or `get_mapgen_object()` was called outside of an `on_generated`
 callback, `nil` is returned.
 
 The following Mapgen objects are currently available:
@@ -4910,20 +4911,27 @@ generated chunk by the current mapgen.
 
 ### `gennotify`
 
-Returns a table mapping requested generation notification types to arrays of
-positions at which the corresponding generated structures are located within
-the current chunk. To enable the capture of positions of interest to be recorded
-call `minetest.set_gen_notify()` first.
+Returns a table. You need to announce your interest in a specific
+field by calling `minetest.set_gen_notify()` *before* map generation happens.
 
-Possible fields of the returned table are:
+* key = string: generation notification type
+* value = list of positions (usually)
+   * Exceptions are denoted in the listing below.
+
+Available generation notification types:
 
 * `dungeon`: bottom center position of dungeon rooms
 * `temple`: as above but for desert temples (mgv6 only)
 * `cave_begin`
 * `cave_end`
 * `large_cave_begin`
 * `large_cave_end`
-* `decoration#id` (see below)
+* `custom`: data originating from [Mapgen environment] (Lua API)
+   * This is a table.
+   * key = user-defined ID (string)
+   * value = arbitrary Lua value
+* `decoration#id`: decorations
+  * (see below)
 
 Decorations have a key in the format of `"decoration#id"`, where `id` is the
 numeric unique decoration ID as returned by `minetest.get_decoration_id()`.
@@ -5587,8 +5595,10 @@ Call these functions only at load time!
 * `minetest.register_on_punchnode(function(pos, node, puncher, pointed_thing))`
     * Called when a node is punched
 * `minetest.register_on_generated(function(minp, maxp, blockseed))`
-    * Called after generating a piece of world. Modifying nodes inside the area
-      is a bit faster than usual.
+    * Called after generating a piece of world between `minp` and `maxp`.
+    * **Avoid using this** whenever possible. As with other callbacks this blocks
+      the main thread and introduces noticable latency.
+      Consider [Mapgen environment] for an alternative.
 * `minetest.register_on_newplayer(function(ObjectRef))`
     * Called when a new player enters the world for the first time
 * `minetest.register_on_punchplayer(function(player, hitter, time_from_last_punch, tool_capabilities, dir, damage))`
@@ -6004,20 +6014,18 @@ Environment access
 * `minetest.get_voxel_manip([pos1, pos2])`
     * Return voxel manipulator object.
     * Loads the manipulator from the map if positions are passed.
-* `minetest.set_gen_notify(flags, {deco_ids})`
+* `minetest.set_gen_notify(flags, [deco_ids], [custom_ids])`
     * Set the types of on-generate notifications that should be collected.
-    * `flags` is a flag field with the available flags:
-        * dungeon
-        * temple
-        * cave_begin
-        * cave_end
-        * large_cave_begin
-        * large_cave_end
-        * decoration
-    * The second parameter is a list of IDs of decorations which notification
+    * `flags`: flag field, see [`gennotify`] for available generation notification types.
+    * The following parameters are optional:
+    * `deco_ids` is a list of IDs of decorations which notification
       is requested for.
+    * `custom_ids` is a list of user-defined IDs (strings) which are
+      requested. By convention these should be the mod name with an optional
+      colon and specifier added, e.g. `"default"` or `"default:dungeon_loot"`
 * `minetest.get_gen_notify()`
-    * Returns a flagstring and a table with the `deco_id`s.
+    * Returns a flagstring, a table with the `deco_id`s and a table with
+      user-defined IDs.
 * `minetest.get_decoration_id(decoration_name)`
     * Returns the decoration ID number for the provided decoration name string,
       or `nil` on failure.
@@ -6573,6 +6581,86 @@ Variables:
     * with all functions and userdata values replaced by `true`, calling any
       callbacks here is obviously not possible
 
+Mapgen environment
+------------------
+
+The engine runs the map generator on separate threads, each of these also has
+a Lua environment. Its primary purpose is to allow mods to operate on newly
+generated parts of the map to e.g. generate custom structures.
+Internally it is referred to as "emerge environment".
+
+Refer to [Async environment] for the usual disclaimer on what environment isolation entails.
+
+The map generator threads, which also contain the above mentioned Lua environment,
+are initialized after all mods have been loaded by the server. After that the
+registered scripts (not all mods!) - see below - are run during initialization of
+the mapgen environment. After that only callbacks happen. The mapgen env
+does not have a global step or timer.
+
+* `minetest.register_mapgen_script(path)`:
+    * Register a path to a Lua file to be imported when a mapgen environment
+      is initialized. Run in order of registration.
+
+### List of APIs exclusive to the mapgen env
+
+* `minetest.register_on_generated(function(vmanip, minp, maxp, blockseed))`
+    * Called after the engine mapgen finishes a chunk but before it is written to
+      the map.
+    * Chunk data resides in `vmanip`. Other parts of the map are not accessible.
+      The area of the chunk if comprised of `minp` and `maxp`, note that is smaller
+      than the emerged area of the VoxelManip.
+      Note: calling `read_from_map()` or `write_to_map()` on the VoxelManipulator object
+      is not necessary and is disallowed.
+    * `blockseed`: 64-bit seed number used for this chunk
+* `minetest.save_gen_notify(id, data)`
+    * Saves data for retrieval using the gennotify mechanism (see [Mapgen objects]).
+    * Data is bound to the chunk that is currently being processed, so this function
+      only makes sense inside the `on_generated` callback.
+    * `id`: user-defined ID (a string)
+      By convention these should be the mod name with an optional
+      colon and specifier added, e.g. `"default"` or `"default:dungeon_loot"`
+    * `data`: any Lua object (will be serialized, no userdata allowed)
+    * returns `true` if the data was remembered. That is if `minetest.set_gen_notify`
+      was called with the same user-defined ID before.
+
+### List of APIs available in the mapgen env
+
+Classes:
+* `AreaStore`
+* `ItemStack`
+* `PerlinNoise`
+* `PerlinNoiseMap`
+* `PseudoRandom`
+* `PcgRandom`
+* `SecureRandom`
+* `VoxelArea`
+* `VoxelManip`
+    * only given by callbacks; cannot access rest of map
+* `Settings`
+
+Functions:
+* Standalone helpers such as logging, filesystem, encoding,
+  hashing or compression APIs
+* `minetest.request_insecure_environment` (same restrictions apply)
+* `minetest.get_biome_id`, `get_biome_name`, `get_heat`, `get_humidity`,
+  `get_biome_data`, `get_mapgen_object`, `get_mapgen_params`, `get_mapgen_edges`,
+  `get_mapgen_setting`, `get_noiseparams`, `get_decoration_id` and more
+* `minetest.get_node`, `set_node`, `find_node_near`, `find_nodes_in_area`,
+  `spawn_tree` and similar
+    * these only operate on the current chunk (if inside a callback)
+
+Variables:
+* `minetest.settings`
+* `minetest.registered_items`, `registered_nodes`, `registered_tools`,
+  `registered_craftitems` and `registered_aliases`
+    * with all functions and userdata values replaced by `true`, calling any
+      callbacks here is obviously not possible
+* `minetest.registered_biomes`, `registered_ores`, `registered_decorations`
+
+Note that node metadata does not exist in the mapgen env, we suggest deferring
+setting any metadata you need to the `on_generated` callback in the regular env.
+You can use the gennotify mechanism to transfer this information.
+
 Server
 ------
 
@@ -7081,10 +7169,6 @@ Global tables
     * Map of registered decoration definitions, indexed by the `name` field.
     * If `name` is nil, the key is the object handle returned by
       `minetest.register_decoration`.
-* `minetest.registered_schematics`
-    * Map of registered schematic definitions, indexed by the `name` field.
-    * If `name` is nil, the key is the object handle returned by
-      `minetest.register_schematic`.
 * `minetest.registered_chatcommands`
     * Map of registered chat command definitions, indexed by name
 * `minetest.registered_privileges`