28× faster reset + complete snapshot capture (fixes goto_depth+snapshot SIGSEGV)

docs/arena_memory_map.txt

@@ -0,0 +1,69 @@
+# nle arena memory map
+arena_base=0x14cabcaee000 used=275904 cap=67108864 obs_dlvl=1 moves=39
+
+## named buffers (arena offset, name) -- sort -n to see layout
+     46864  gbuf
+     16288  context
+      9120  obufs
+     45200  mbufs
+      8208  disco
+    154384  blstats
+    194160  level(struct)
+    234528  rooms
+    252240  doors
+    252480  level_info
+    252992  lastseentyp
+      4512  youmonst
+      4656  mvitals
+      5808  killer
+      4144  spl_book
+      4128  quest_status
+     19776  objects
+     37936  obj_descr
+     16624  rndmonst
+     16960  artilist
+    254800  muse_m
+    172048  could_see
+    175408  viz_clear
+    177088  left_ptrs
+    178768  right_ptrs
+    184176  wheads
+    184432  wtails
+    184688  wgrowtime
+     12192  tty_status
+         0  tcap
+      2992  topology
+      3056  dungeons
+
+## monsters fmon (offset id mnum hp species)
+    275264  id=54 mnum=16 hp=8 little dog
+    273776  id=39 mnum=318 hp=4 newt
+    273632  id=38 mnum=58 hp=4 kobold
+
+## objects fobj (offset id otyp)
+    275168  id=53 otyp=410
+    274976  id=51 otyp=410
+    274880  id=50 otyp=436
+    274784  id=49 otyp=410
+    274688  id=48 otyp=437
+    274592  id=47 otyp=442
+    274400  id=45 otyp=410
+    274208  id=43 otyp=435
+    274016  id=41 otyp=189
+    273920  id=40 otyp=189
+    273536  id=37 otyp=287
+    273392  id=36 otyp=240
+    273296  id=35 otyp=62
+    273200  id=34 otyp=1
+    272784  id=31 otyp=410
+    272688  id=30 otyp=410
+    272592  id=29 otyp=410
+    272496  id=28 otyp=410
+    272400  id=27 otyp=308
+    272304  id=26 otyp=447
+    272208  id=25 otyp=447
+
+## grid monster ptrs (x y offset in_fmon valid_data)
+ 32   9      275264  in_fmon=1 valid_data=1
+ 38  17      273632  in_fmon=1 valid_data=1
+ 45  12      273776  in_fmon=1 valid_data=1

docs/game_memory_map.txt

@@ -0,0 +1,77 @@
+# nle whole-game memory map
+obs_dlvl=1 moves=39
+
+## whole-game regions (addr  size  what)
+         0x127ca60       28376  nle_ctx_t struct (fixed per-env state)
+    0x14beb7aee000      275904  arena (used; cap=67108864) -- all dynamic state
+    0x14bf44dc3000       65536  coroutine stack (fcontext)
+         0x128f650      141015  rl display mirror (libc, outside arena)
+
+## arena named buffers + chains (offsets relative to arena_base=0x14beb7aee000)
+
+## named buffers (arena offset, name) -- sort -n to see layout
+     46864  gbuf
+     16288  context
+      9120  obufs
+     45200  mbufs
+      8208  disco
+    154384  blstats
+    194160  level(struct)
+    234528  rooms
+    252240  doors
+    252480  level_info
+    252992  lastseentyp
+      4512  youmonst
+      4656  mvitals
+      5808  killer
+      4144  spl_book
+      4128  quest_status
+     19776  objects
+     37936  obj_descr
+     16624  rndmonst
+     16960  artilist
+    254800  muse_m
+    172048  could_see
+    175408  viz_clear
+    177088  left_ptrs
+    178768  right_ptrs
+    184176  wheads
+    184432  wtails
+    184688  wgrowtime
+     12192  tty_status
+         0  tcap
+      2992  topology
+      3056  dungeons
+
+## monsters fmon (offset id mnum hp species)
+    275264  id=54 mnum=16 hp=8 little dog
+    273776  id=39 mnum=318 hp=4 newt
+    273632  id=38 mnum=58 hp=4 kobold
+
+## objects fobj (offset id otyp)
+    275168  id=53 otyp=410
+    274976  id=51 otyp=410
+    274880  id=50 otyp=436
+    274784  id=49 otyp=410
+    274688  id=48 otyp=437
+    274592  id=47 otyp=442
+    274400  id=45 otyp=410
+    274208  id=43 otyp=435
+    274016  id=41 otyp=189
+    273920  id=40 otyp=189
+    273536  id=37 otyp=287
+    273392  id=36 otyp=240
+    273296  id=35 otyp=62
+    273200  id=34 otyp=1
+    272784  id=31 otyp=410
+    272688  id=30 otyp=410
+    272592  id=29 otyp=410
+    272496  id=28 otyp=410
+    272400  id=27 otyp=308
+    272304  id=26 otyp=447
+    272208  id=25 otyp=447
+
+## grid monster ptrs (x y offset in_fmon valid_data)
+ 32   9      275264  in_fmon=1 valid_data=1
+ 38  17      273632  in_fmon=1 valid_data=1
+ 45  12      273776  in_fmon=1 valid_data=1

environments/nethack/nethack_core/_engine.py

@@ -166,6 +166,11 @@ class NleSettings(ctypes.Structure):
         ("tune_n", ctypes.c_int),
         ("tune_idx", ctypes.c_int * NLE_TUNE_MAX),
         ("tune_val", ctypes.c_double * NLE_TUNE_MAX),
+        # Optional shared read-only data dir. When set, the immutable game data
+        # (DLB nhdat, data/oracles/rumors, config) is read from here and only the
+        # writable game-state files live under hackdir — so no per-reset copy of
+        # the ~3.7MB dat tree is needed. Empty => every prefix uses hackdir.
+        ("datadir", ctypes.c_char * 256),
     ]
 
 
@@ -268,7 +273,15 @@ def __init__(self) -> None:
         self._obs.misc = self._misc.ctypes.data_as(ctypes.POINTER(ctypes.c_int))
 
         self._ctx = None
-        self._hackdir = None  # tempfile.mkdtemp() path string
+        #: Tiny per-env WRITABLE directory (level/save/bones/score/lock files),
+        #: created once and reused across games. The large read-only game data is
+        #: read directly from the shared source dat (settings.datadir), so this
+        #: holds only the few small game-state files — no 3.7MB copy per reset.
+        self._hackdir = None
+        #: Writable templates the engine appends to (scores/logs/lock). Seeded
+        #: empty in the writable dir on each reset so every game starts clean —
+        #: matching the empty templates the old per-reset copytree provided.
+        self._writable_templates = ("record", "logfile", "xlogfile", "perm")
 
         # Outstanding snapshot handles created by this instance.  A handle is
         # bound to the ctx that created it; end() frees any the caller leaked.
@@ -379,6 +392,12 @@ def _setup_argtypes(self) -> None:
         ]
         lib.nle_set_seed.restype = None
 
+        # Debug: dump the per-env arena memory map (named buffers, fmon/fobj
+        # chains, and the monster grid with fmon-membership) to a file. A
+        # diagnostic tool for arena-reuse / dangling-pointer investigations.
+        lib.nle_dbg_memmap.argtypes = [ctypes.c_void_p, ctypes.c_char_p]
+        lib.nle_dbg_memmap.restype = None
+
     def _build_dat_path(self) -> Path:
         """Return the path to the pre-built dat directory (NetHack data files).
 
@@ -424,18 +443,21 @@ def start(self, core: int, disp: int, tune: dict = None,
         player-name suffix (e.g. "Val-hum-neu-fem" for a female neutral human
         Valkyrie). If None, the historical default (Monk) is used.
         """
-        # Tear down any prior game before creating a new one.
-        self.end()
+        # Tear down any prior game's C context (the writable hackdir is reused).
+        self._teardown_ctx()
 
-        # Make a writable copy of the built dat directory so the engine can
-        # write lock/record/level files without polluting the source tree.
-        src_dat = self._build_dat_path()
-        self._hackdir = tempfile.mkdtemp(prefix="nethack_hackdir_")
-        shutil.copytree(str(src_dat), self._hackdir, dirs_exist_ok=True)
+        # Provide a fresh, tiny writable hackdir for this game's game-state files.
+        # The large read-only data is read directly from the shared source dat
+        # (settings.datadir below), so reset no longer copies the ~3.7MB dat tree
+        # — historically the dominant (~29ms) reset cost.
+        self._ensure_hackdir()
 
         # Build settings.
         settings = NleSettings()
         settings.hackdir = self._hackdir.encode()
+        # Shared read-only data dir: the engine reads nhdat/data/oracles/config
+        # from here (never writes), so many envs share it with zero per-env copy.
+        settings.datadir = str(self._build_dat_path()).encode()
         settings.scoreprefix = b""
         char = character if character is not None else _DEFAULT_CHARACTER
         options_bytes = (_OPTIONS_BASE + ",name:Agent-" + char).encode()
@@ -479,21 +501,51 @@ def step(self, action: int) -> "RawEngine":
         self._lib.nle_step(self._ctx, ctypes.byref(self._obs))
         return self
 
-    def end(self) -> None:
-        """Tear down the current game context and clean up the temp hackdir.
+    def _teardown_ctx(self) -> None:
+        """Free the current game's C context and snapshot handles (keep hackdir).
 
-        Frees any outstanding snapshot handles first: they become invalid once
-        the ctx they would restore into is gone, and freeing them here prevents
-        leaks across games (start() calls end() before creating a new game).
-        Handles are self-contained copies, so destroy is independent of the ctx
-        and ordering relative to nle_end does not matter.
+        Snapshot handles become invalid once the ctx they restore into is gone,
+        so they are freed here; they are self-contained copies, so destroy is
+        independent of the ctx and ordering relative to nle_end does not matter.
         """
         for snap in list(self._snapshots):
             self._lib.nle_fr_destroy(snap)
         self._snapshots.clear()
         if self._ctx is not None:
             self._lib.nle_end(self._ctx)
             self._ctx = None
+
+    def _ensure_hackdir(self) -> None:
+        """Provide a fresh, empty-but-seeded writable hackdir for a new game.
+
+        The directory is created once per engine and reused: on each reset its
+        prior game-state files (level/save/bones/lock plus the score/log files)
+        are removed and the empty writable templates re-seeded. It holds only a
+        handful of small files — the bulk read-only data is read from the shared
+        datadir — so this is sub-millisecond regardless of how the prior game
+        grew. The seeded empty templates reproduce exactly what the old
+        per-reset copytree placed here.
+        """
+        if self._hackdir is None:
+            self._hackdir = tempfile.mkdtemp(prefix="nethack_hackdir_")
+        else:
+            for name in os.listdir(self._hackdir):
+                p = os.path.join(self._hackdir, name)
+                if os.path.isdir(p):
+                    shutil.rmtree(p, ignore_errors=True)
+                else:
+                    os.unlink(p)
+        for tmpl in self._writable_templates:
+            open(os.path.join(self._hackdir, tmpl), "wb").close()
+
+    def end(self) -> None:
+        """Tear down the current game context and remove the temp hackdir.
+
+        Called on close()/__del__. Within a single engine, start() reuses the
+        hackdir across games (see _ensure_hackdir) and only this final teardown
+        removes it.
+        """
+        self._teardown_ctx()
         if self._hackdir is not None:
             shutil.rmtree(self._hackdir, ignore_errors=True)
             self._hackdir = None
@@ -585,6 +637,21 @@ def reseed(self, core: int, disp: int) -> "RawEngine":
         )
         return self
 
+    def memmap(self, path: str) -> str:
+        """Dump the per-env arena memory map to ``path`` (debug tool).
+
+        Writes: arena base/used/cap; every named per-env buffer by arena offset;
+        the live monster (fmon) and object (fobj) chains; and the monster grid
+        (level.monsters[x][y]) with fmon-membership + data-validity per cell.
+        Lets you classify any arena pointer and spot dangling/stale pointers
+        (grid entries not in fmon, monsters with out-of-range ``data``). Useful
+        for arena-reuse / snapshot corruption debugging. Returns ``path``.
+        """
+        if self._ctx is None:
+            raise RuntimeError("memmap() requires an active game; call start() first")
+        self._lib.nle_dbg_memmap(self._ctx, str(path).encode())
+        return path
+
     # ------------------------------------------------------------------
     # Portable level blob save / load
     # ------------------------------------------------------------------