fix: route GPU backend environments by DP rank

docs/sphinx/source/en/2-user_guide/2-algorithms/1-ppo.md

@@ -50,6 +50,13 @@ device. Do not set `training.device` and `training.devices` together. The
 configured order is preserved, including when the parent already has
 `CUDA_VISIBLE_DEVICES` set.
 
+For the IsaacGym, IsaacSim, and Genesis owners, the same topology is also
+applied to the simulator environment. Torchrun workers receive the local
+index inside their remapped `CUDA_VISIBLE_DEVICES` list (for example, host
+device 5 is sent as `device_id=1` when the worker sees `[4,5]`); off-policy
+workers keep the parent process's visible index namespace. Genesis selects
+its process-wide session before `gs.init`.
+
 `algo.num_envs` is a **per-rank** count, not a global budget. For `W` ranks,
 `N` configured envs, and rollout length `T`:
 

docs/sphinx/source/en/2-user_guide/2-algorithms/3-sac.md

@@ -54,6 +54,11 @@ materialization. The collector therefore does not fall back to Warp's fresh-proc
 of `cuda:0`. The local binding is recorded as `collector_backend_device` in the runtime
 manifest.
 
+IsaacGym, IsaacSim, and Genesis receive the rank-selected simulator device
+through the environment override as well. Off-policy collectors use the
+parent's visible CUDA indices; Genesis binds its process-wide session before
+initialization.
+
 MuJoCo has a committed multi-GPU scaling benchmark. The mjwarp per-rank placement contract is
 covered by `tests/base/backend/test_process_device.py` and the off-policy runner/worker unit
 tests; the repository does not currently contain an mjwarp multi-GPU throughput or convergence

docs/sphinx/source/zh_CN/2-user_guide/2-algorithms/1-ppo.md

@@ -48,6 +48,11 @@ uv run train --algo ppo --task g1_motion_tracking --sim mujoco \
 `training.devices`。父进程已有 `CUDA_VISIBLE_DEVICES` 时，配置索引仍按父进程可见
 设备解释，并保留用户给定顺序。
 
+对 IsaacGym、IsaacSim 和 Genesis owner，同一拓扑也会传给环境仿真器。torchrun
+worker 继承重映射后的 `CUDA_VISIBLE_DEVICES`，因此传给 worker 的是本地索引（例如
+worker 看到 `[4,5]` 时，主机设备 5 传为 `device_id=1`）；off-policy worker 保持父进程
+可见设备索引。Genesis 会在 `gs.init` 前选择每个进程的 session 设备。
+
 `algo.num_envs` 是**每个 rank** 的环境数，不是全局预算。设 rank 数为 `W`、配置
 环境数为 `N`、rollout 长度为 `T`：
 

docs/sphinx/source/zh_CN/2-user_guide/2-algorithms/3-sac.md

@@ -97,6 +97,8 @@ MuJoCo worker 线程逐核绑定外，collector 进程本身（含 Numba 并行
 - MuJoCo 有已提交的多卡 scaling benchmark；mjwarp 的 per-rank device placement 有
   `tests/base/backend/test_process_device.py` 与 off-policy runner/worker 单测覆盖，但仓库中
   尚无 mjwarp 多卡吞吐或收敛 benchmark。
+- IsaacGym、IsaacSim 和 Genesis 的 collector 环境也会收到 rank 对应的仿真设备。off-policy
+  使用父进程可见的 CUDA 索引；Genesis 在初始化 `gs.init` 前绑定进程级 session 设备。
 - 仅单节点：rank 之间通过 run 目录里的 FileStore rendezvous，NCCL 走 TCP
   loopback（默认 `NCCL_P2P_DISABLE=1` / `NCCL_SHM_DISABLE=1`，环境变量显式设置
   时优先）——部分机型（如 RTX 6000D）的 NCCL P2P/SHM peer transport 不可靠，

src/unilab/base/backend_factory.py

@@ -14,15 +14,35 @@
 from unisim.backend.base import SimBackend
 
 from unilab.assets.hub import ensure_robot_assets_for_paths
+from unilab.base.process_device import bind_genesis_process_device
 
 if TYPE_CHECKING:
     from unilab.base.base import EnvCfg
     from unilab.base.scene import SceneCfg
 
 
+def _legacy_genesis_device_option_error(exc: TypeError) -> bool:
+    """Identify an old UniSim adapter rejecting the optional device keyword.
+
+    UniSim 1.1 reports unknown backend options from ``GenesisBackend`` while
+    other compatible releases may expose Python's usual ``unexpected keyword``
+    wording.  Keep the compatibility retry narrowly scoped to those messages;
+    constructor errors from the actual Genesis runtime must still propagate.
+    """
+
+    message = str(exc).lower()
+    mentions_device = "genesis_device_id" in message or "device_id" in message
+    rejects_keyword = (
+        "does not accept backend options" in message
+        or "unexpected keyword argument" in message
+        or "unexpected keyword" in message
+    )
+    return mentions_device and rejects_keyword
+
+
 def env_backend_kwargs(cfg: "EnvCfg") -> dict[str, Any]:
     """Translate ``EnvCfg`` backend knobs into UniSim adapter options."""
-    return {
+    result: dict[str, Any] = {
         "post_step_forward_sensor": cfg.post_step_forward_sensor,
         "motrix_max_iterations": cfg.motrix_max_iterations,
         "chunk_size": cfg.chunk_size,
@@ -45,6 +65,13 @@ def env_backend_kwargs(cfg: "EnvCfg") -> dict[str, Any]:
         "isaacsim_render_width": cfg.isaacsim_render_width,
         "isaacsim_render_height": cfg.isaacsim_render_height,
     }
+    # Keep the optional key absent for legacy unisim-core releases that do not
+    # know about Genesis' explicit device argument.  Once a rank selects a
+    # device the key is added below and ``create_backend`` supplies a narrow
+    # compatibility fallback for those releases.
+    if cfg.genesis_device_id is not None:
+        result["genesis_device_id"] = cfg.genesis_device_id
+    return result
 
 
 def create_backend(
@@ -63,7 +90,38 @@ def create_backend(
         [scene.model_file, scene.visual_model_file, *scene.fragment_files]
     )
     kwargs["body_state_required"] = body_state_required
-    return unisim.create_backend(backend_type, scene, num_envs, sim_dt, **kwargs)
+    if backend_type == "genesis" and kwargs.get("genesis_device_id") is not None:
+        # Bind before any unisim-core Genesis constructor can call gs.init.
+        # New unisim-core releases repeat this idempotently; old releases do
+        # not accept the keyword, so the retry below still gets the correct
+        # process-wide device.  Binding a non-zero id pins
+        # CUDA_VISIBLE_DEVICES (Quadrants only honors the first visible
+        # device), so forward the *post-pin* in-process index.
+        genesis_device_id = kwargs["genesis_device_id"]
+        if (
+            isinstance(genesis_device_id, bool)
+            or not isinstance(genesis_device_id, int)
+            or genesis_device_id < 0
+        ):
+            raise ValueError(
+                "genesis_device_id must be a non-negative integer or None, "
+                f"got {genesis_device_id!r}"
+            )
+        bound = bind_genesis_process_device(f"cuda:{genesis_device_id}")
+        kwargs["genesis_device_id"] = int(bound.rsplit(":", 1)[1])
+    try:
+        return unisim.create_backend(backend_type, scene, num_envs, sim_dt, **kwargs)
+    except TypeError as exc:
+        if backend_type != "genesis" or "genesis_device_id" not in kwargs:
+            raise
+        # unisim-core < 1.2 has no Genesis device field and reports the
+        # unknown option from GenesisBackend.  Retry only for that precise
+        # capability error; unrelated constructor TypeErrors must propagate.
+        if not _legacy_genesis_device_option_error(exc):
+            raise
+        legacy_kwargs = dict(kwargs)
+        legacy_kwargs.pop("genesis_device_id", None)
+        return unisim.create_backend(backend_type, scene, num_envs, sim_dt, **legacy_kwargs)
 
 
 __all__ = ["SimBackend", "create_backend", "env_backend_kwargs"]

src/unilab/base/base.py

@@ -56,6 +56,10 @@ class EnvCfg:
     # backend defaults (device 0, generous handshake/step timeout).
     isaacgym_device_id: Optional[int] = None
     isaacgym_worker_timeout_s: Optional[float] = None
+    # ``genesis`` owns one process-wide GPU session.  The explicit device id
+    # must be selected before ``gs.init`` so each data-parallel rank gets its
+    # own simulator device; ``None`` keeps Genesis' current/default device.
+    genesis_device_id: Optional[int] = None
     # ``genesis`` drops the MJCF global <option> block at import (REPORT #1372
     # §3.3), so integrator / constraint solver / friction cone / solver
     # iterations must be explicit owner fields. ``None`` keeps the Genesis
@@ -123,6 +127,15 @@ def validate(self):
                 "isaacgym_worker_timeout_s must be a positive number or None, "
                 f"got {self.isaacgym_worker_timeout_s!r}"
             )
+        if self.genesis_device_id is not None and (
+            isinstance(self.genesis_device_id, bool)
+            or not isinstance(self.genesis_device_id, int)
+            or self.genesis_device_id < 0
+        ):
+            raise ValueError(
+                "genesis_device_id must be a non-negative integer or None, "
+                f"got {self.genesis_device_id!r}"
+            )
         for name, value in (
             ("genesis_integrator", self.genesis_integrator),
             ("genesis_constraint_solver", self.genesis_constraint_solver),

src/unilab/base/env_factory.py

@@ -16,6 +16,8 @@
 
 from uni_rl.env_contract import EnvFactory, EnvProtocol
 
+from unilab.base.process_device import bind_genesis_process_device
+
 
 def make_registry_env(
     task_name: str,
@@ -34,6 +36,32 @@ def make_registry_env(
     from unilab.base.registry import ensure_registries
 
     ensure_registries()
+    # Genesis owns a process-wide session whose Quadrants runtime binds the
+    # first entry of CUDA_VISIBLE_DEVICES.  Off-policy/APPO collectors are
+    # fresh spawn processes, so the parent-side binding cannot reach them;
+    # carry the explicit cold-path id in the opaque override and bind
+    # immediately before registry construction.  Binding a non-zero id pins
+    # CUDA_VISIBLE_DEVICES for this process, so forward the post-pin
+    # in-process index downstream.  Newer unisim-core versions repeat this
+    # check in GenesisBackend itself, making this compatibility guard
+    # idempotent.
+    if sim_backend == "genesis" and env_cfg_override is not None:
+        genesis_device_id = env_cfg_override.get("genesis_device_id")
+        if genesis_device_id is not None:
+            if (
+                isinstance(genesis_device_id, bool)
+                or not isinstance(genesis_device_id, int)
+                or genesis_device_id < 0
+            ):
+                raise ValueError(
+                    "genesis_device_id must be a non-negative integer or None, "
+                    f"got {genesis_device_id!r}"
+                )
+            bound = bind_genesis_process_device(f"cuda:{genesis_device_id}")
+            env_cfg_override = {
+                **env_cfg_override,
+                "genesis_device_id": int(bound.rsplit(":", 1)[1]),
+            }
     # ABEnv satisfies EnvProtocol at runtime (reset/set_nan_guard live on
     # NpEnv); the declared ABEnv type predates the uni_rl protocol.
     return cast(