revert PR 1658 and add more docstring for random seed

alf/algorithms/config.py

@@ -111,7 +111,15 @@ def __init__(self,
                 normalized whereas hindsight data directly pulled from the replay buffer
                 will not be normalized.  Data will be in mismatch, causing training to
                 suffer and potentially fail.
-            random_seed (None|int): random seed, a random seed is used if None
+            random_seed (None|int): random seed, a random seed is used if None.
+                For a None random seed, all DDP ranks (if multi-gpu training used)
+                will have a None random seed set to their ``TrainerConfig.random_seed``.
+                This means that the actual random seed used by each rank is purely
+                random. A None random seed won't set a deterministic torch behavior.
+                If a specific random seed is set, DDP rank>0 (if multi-gpu training
+                used) will have a random seed set to a value that is deterministically
+                "randomized" from this random seed. In this case, all ranks will
+                have a deterministic torch behavior.
             num_iterations (int): For RL trainer, indicates number of update
                 iterations (ignored if 0). Note that for off-policy algorithms, if
                 ``initial_collect_steps>0``, then the first

alf/config_helpers.py

@@ -270,8 +270,12 @@ def get_env():
         # The random.seed(random_seed) is temporary and will be overridden by
         # set_random_seed() below.
         random.seed(random_seed)
-        for _ in range(PerProcessContext().ddp_rank):
-            random_seed = random.randint(0, 2**32)
+
+        if random_seed is not None:
+            # If random seed is None, we will have None for other ranks, too.
+            # A 'None' random seed won't set a deterministic torch behavior.
+            for _ in range(PerProcessContext().ddp_rank):
+                random_seed = random.randint(0, 2**32)
         config1("TrainerConfig.random_seed", random_seed, raise_if_used=False)
 
         # We have to call set_random_seed() here because we need the actual