diff --git a/gym/benchmarks/__init__.py b/gym/benchmarks/__init__.py
index 73f32319d00..6d744d4d72b 100644
--- a/gym/benchmarks/__init__.py
+++ b/gym/benchmarks/__init__.py
@@ -11,49 +11,49 @@
     description='7 Atari games, with pixel observations',
     tasks=[
         {
-            'env_id': 'BeamRiderNoFrameskip-v3',
+            'env_id': 'BeamRiderNoFrameskip-v4',
             'trials': 2,
             'max_timesteps': int(2e8),
             'reward_floor':   363.9,
             'reward_ceiling': 60000.0,
         },
         {
-            'env_id': 'BreakoutNoFrameskip-v3',
+            'env_id': 'BreakoutNoFrameskip-v4',
             'trials': 2,
             'max_timesteps': int(2e8),
             'reward_floor':   1.7,
             'reward_ceiling': 800.0,
         },
         {
-            'env_id': 'EnduroNoFrameskip-v3',
+            'env_id': 'EnduroNoFrameskip-v4',
             'trials': 2,
             'max_timesteps': int(2e8),
             'reward_floor':   0.0,
             'reward_ceiling': 5000.0,
         },
         {
-            'env_id': 'PongNoFrameskip-v3',
+            'env_id': 'PongNoFrameskip-v4',
             'trials': 2,
             'max_timesteps': int(2e8),
             'reward_floor':  -20.7,
             'reward_ceiling': 21.0,
         },
         {
-            'env_id': 'QbertNoFrameskip-v3',
+            'env_id': 'QbertNoFrameskip-v4',
             'trials': 2,
             'max_timesteps': int(2e8),
             'reward_floor':   163.9,
             'reward_ceiling': 40000.0,
         },
         {
-            'env_id': 'SeaquestNoFrameskip-v3',
+            'env_id': 'SeaquestNoFrameskip-v4',
             'trials': 2,
             'max_timesteps': int(2e8),
             'reward_floor':   68.4,
             'reward_ceiling': 100000.0,
         },
         {
-            'env_id': 'SpaceInvadersNoFrameskip-v3',
+            'env_id': 'SpaceInvadersNoFrameskip-v4',
             'trials': 2,
             'max_timesteps': int(2e8),
             'reward_floor':   148.0,
@@ -69,49 +69,49 @@
     description='7 Atari games, with pixel observations',
     tasks=[
         {
-            'env_id': 'BeamRiderNoFrameskip-v3',
+            'env_id': 'BeamRiderNoFrameskip-v4',
             'trials': 2,
             'max_timesteps': int(4e7),
             'reward_floor':   363.9,
             'reward_ceiling': 60000.0,
         },
         {
-            'env_id': 'BreakoutNoFrameskip-v3',
+            'env_id': 'BreakoutNoFrameskip-v4',
             'trials': 2,
             'max_timesteps': int(4e7),
             'reward_floor':   1.7,
             'reward_ceiling': 800.0,
         },
         {
-            'env_id': 'EnduroNoFrameskip-v3',
+            'env_id': 'EnduroNoFrameskip-v4',
             'trials': 2,
             'max_timesteps': int(4e7),
             'reward_floor':   0.0,
             'reward_ceiling': 5000.0,
         },
         {
-            'env_id': 'PongNoFrameskip-v3',
+            'env_id': 'PongNoFrameskip-v4',
             'trials': 2,
             'max_timesteps': int(4e7),
             'reward_floor':  -20.7,
             'reward_ceiling': 21.0,
         },
         {
-            'env_id': 'QbertNoFrameskip-v3',
+            'env_id': 'QbertNoFrameskip-v4',
             'trials': 2,
             'max_timesteps': int(4e7),
             'reward_floor':   163.9,
             'reward_ceiling': 40000.0,
         },
         {
-            'env_id': 'SeaquestNoFrameskip-v3',
+            'env_id': 'SeaquestNoFrameskip-v4',
             'trials': 2,
             'max_timesteps': int(4e7),
             'reward_floor':   68.4,
             'reward_ceiling': 100000.0,
         },
         {
-            'env_id': 'SpaceInvadersNoFrameskip-v3',
+            'env_id': 'SpaceInvadersNoFrameskip-v4',
             'trials': 2,
             'max_timesteps': int(4e7),
             'reward_floor':   148.0,
@@ -127,49 +127,49 @@
     description='7 Atari games, with pixel observations',
     tasks=[
         {
-            'env_id': 'FreewayNoFrameskip-v3',
+            'env_id': 'FreewayNoFrameskip-v4',
             'trials': 2,
             'max_timesteps': int(4e7),
             'reward_floor':   0.1,
             'reward_ceiling': 31.0,
         },
         {
-            'env_id': 'GravitarNoFrameskip-v3',
+            'env_id': 'GravitarNoFrameskip-v4',
             'trials': 2,
             'max_timesteps': int(4e7),
             'reward_floor':   245.5,
             'reward_ceiling': 1000.0,
         },
         {
-            'env_id': 'MontezumaRevengeNoFrameskip-v3',
+            'env_id': 'MontezumaRevengeNoFrameskip-v4',
             'trials': 2,
             'max_timesteps': int(4e7),
             'reward_floor':   25.0,
             'reward_ceiling': 10000.0,
         },
         {
-            'env_id': 'PitfallNoFrameskip-v3',
+            'env_id': 'PitfallNoFrameskip-v4',
             'trials': 2,
             'max_timesteps': int(4e7),
             'reward_floor':  -348.8,
             'reward_ceiling': 1000.0,
         },
         {
-            'env_id': 'PrivateEyeNoFrameskip-v3',
+            'env_id': 'PrivateEyeNoFrameskip-v4',
             'trials': 2,
             'max_timesteps': int(4e7),
             'reward_floor':   662.8,
             'reward_ceiling': 100.0,
         },
         {
-            'env_id': 'SolarisNoFrameskip-v3',
+            'env_id': 'SolarisNoFrameskip-v4',
             'trials': 2,
             'max_timesteps': int(4e7),
             'reward_floor':   2047.2,
             'reward_ceiling': 5000.0,
         },
         {
-            'env_id': 'VentureNoFrameskip-v3',
+            'env_id': 'VentureNoFrameskip-v4',
             'trials': 2,
             'max_timesteps': int(4e7),
             'reward_floor':   18.0,
diff --git a/gym/envs/__init__.py b/gym/envs/__init__.py
index e802aaeaddd..b22198a4f74 100644
--- a/gym/envs/__init__.py
+++ b/gym/envs/__init__.py
@@ -322,7 +322,7 @@
         )
 
         register(
-            id='{}-v3'.format(name),
+            id='{}-v4'.format(name),
             entry_point='gym.envs.atari:AtariEnv',
             kwargs={'game': game, 'obs_type': obs_type},
             max_episode_steps=100000,
@@ -345,7 +345,7 @@
         )
 
         register(
-            id='{}Deterministic-v3'.format(name),
+            id='{}Deterministic-v4'.format(name),
             entry_point='gym.envs.atari:AtariEnv',
             kwargs={'game': game, 'obs_type': obs_type, 'frameskip': frameskip},
             max_episode_steps=100000,
@@ -363,7 +363,7 @@
         # No frameskip. (Atari has no entropy source, so these are
         # deterministic environments.)
         register(
-            id='{}NoFrameskip-v3'.format(name),
+            id='{}NoFrameskip-v4'.format(name),
             entry_point='gym.envs.atari:AtariEnv',
             kwargs={'game': game, 'obs_type': obs_type, 'frameskip': 1}, # A frameskip of 1 means we get every frame
             max_episode_steps=frameskip * 100000,
diff --git a/gym/envs/tests/rollout.json b/gym/envs/tests/rollout.json
index a4191957529..470f9bb0969 100644
--- a/gym/envs/tests/rollout.json
+++ b/gym/envs/tests/rollout.json
@@ -11,7 +11,7 @@
     "observations": "c176c4e28e45872e47c9efd1ad0b203e202aae59122cb541790c14bc745d0a55",
     "rewards": "49bf48ac6831ac4bfa478b736f8b6e7700dbc4c247590fd4fa1bb736ea69bcf2"
   },
-  "AirRaid-ram-v3": {
+  "AirRaid-ram-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "4d0f78406d45584957836fc46c75c850d6882141781984263b87f41e1ec87a09",
@@ -23,7 +23,7 @@
     "observations": "be2e4157a0c067039fd21b85588261d33d0390127a9e5d1ed3c17a02b1a5d869",
     "rewards": "b9b6ab4235bedba68b200434e7967ee00f60cee40065b3f26bb1913500bdce8b"
   },
-  "AirRaid-ramDeterministic-v3": {
+  "AirRaid-ramDeterministic-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "e370fbaa3abba7e26cbf586c4e603a142809d8fe1c750763c0f49f0a667a9259",
@@ -35,7 +35,7 @@
     "observations": "7241af6e4aa560dbe637bcb6ac5b5e52d06a72739842c2a70bc0df7915298d0c",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "AirRaid-ramNoFrameskip-v3": {
+  "AirRaid-ramNoFrameskip-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "a5183bbf11ace53d7605224f00afe27f24ca9242ca197e35f527f7fbe9c3a15b",
@@ -47,7 +47,7 @@
     "observations": "84ee0f67c28dd412a106257756a5ec8d7b5b3b7b23e0df2377f83f99b1a17e39",
     "rewards": "49bf48ac6831ac4bfa478b736f8b6e7700dbc4c247590fd4fa1bb736ea69bcf2"
   },
-  "AirRaid-v3": {
+  "AirRaid-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "dcbe8457aca9612161b0ce87fe6c3affff3cc2a5dc754575c2e9da20556a052a",
@@ -59,7 +59,7 @@
     "observations": "716d3ce120ebec64e56c98a08832541ba1e75f00460e745ea59c32a4300ac800",
     "rewards": "b9b6ab4235bedba68b200434e7967ee00f60cee40065b3f26bb1913500bdce8b"
   },
-  "AirRaidDeterministic-v3": {
+  "AirRaidDeterministic-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "716d3ce120ebec64e56c98a08832541ba1e75f00460e745ea59c32a4300ac800",
@@ -71,7 +71,7 @@
     "observations": "f845a473873a5adb64e05a58d94e08fa2c3057e6549251e876a5bbdef20ca7df",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "AirRaidNoFrameskip-v3": {
+  "AirRaidNoFrameskip-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "f845a473873a5adb64e05a58d94e08fa2c3057e6549251e876a5bbdef20ca7df",
@@ -83,7 +83,7 @@
     "observations": "288b69bb2153553b11392da314b8908441f3ad9f40bfbdeabefc47fd5e9d8b1a",
     "rewards": "9abfc5f67ff4d5b1c61fea6dcfa39f60accd6a462a502808c72cbebfb14330fc"
   },
-  "Alien-ram-v3": {
+  "Alien-ram-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "bba25b18c3fdded610b12fe906cd9af5a52f0b13a243b2a7d792b119b26653c4",
@@ -95,7 +95,7 @@
     "observations": "528837875963080bf52a2a81fd30342950a51989098d80f1bdad066e7c24482a",
     "rewards": "c657db058b5f0de868e03af71b5be9e7ebc18c2d7f01af65b42af0ee9818d544"
   },
-  "Alien-ramDeterministic-v3": {
+  "Alien-ramDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "527b9e205297a11e2be43956368d15db1c3efe02b1f6333608bb1c1eda6ca97e",
@@ -107,7 +107,7 @@
     "observations": "a0e85058b611132fe59b429d0e6de1eb1d1fdea5054a35c3d8cdabf91566698e",
     "rewards": "7145e9f12177823a754dfd6eb2d701248027398726dabe8400b7b59eddd3393b"
   },
-  "Alien-ramNoFrameskip-v3": {
+  "Alien-ramNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "09a6be6669a18d10e68ebdb0103c3dcc43dad93a181b6228ccdf50a26dbd42ea",
@@ -119,7 +119,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "9abfc5f67ff4d5b1c61fea6dcfa39f60accd6a462a502808c72cbebfb14330fc"
   },
-  "Alien-v3": {
+  "Alien-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -131,7 +131,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "c657db058b5f0de868e03af71b5be9e7ebc18c2d7f01af65b42af0ee9818d544"
   },
-  "AlienDeterministic-v3": {
+  "AlienDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -143,7 +143,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "7145e9f12177823a754dfd6eb2d701248027398726dabe8400b7b59eddd3393b"
   },
-  "AlienNoFrameskip-v3": {
+  "AlienNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -155,7 +155,7 @@
     "observations": "985140125ce7ba40c218c2109e6bbb8bad052149f8bd319a2a49793edf32cf6e",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Amidar-ram-v3": {
+  "Amidar-ram-v4": {
     "actions": "b199b81b77e4e2a8aad9a5663315bd9f7a65ba9ad191c7f8645848e7291df62e",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "6388772a0efd69d5f987d37509abfa2ee7a7e8917dc828cc729f82ac986c612c",
@@ -167,7 +167,7 @@
     "observations": "ea35c0db591b1c7593a852793a68be2a389d077d3815301ee39de6d30fc8442b",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Amidar-ramDeterministic-v3": {
+  "Amidar-ramDeterministic-v4": {
     "actions": "b199b81b77e4e2a8aad9a5663315bd9f7a65ba9ad191c7f8645848e7291df62e",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "afdb951168e1ec283f263065cff9dabaf99debdd50d6a832a45669a8d768a8e8",
@@ -179,7 +179,7 @@
     "observations": "bdc927c67b2a793b4383103d0c4af9dbc4c8535d943e8217084403f24c1b87d0",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Amidar-ramNoFrameskip-v3": {
+  "Amidar-ramNoFrameskip-v4": {
     "actions": "b199b81b77e4e2a8aad9a5663315bd9f7a65ba9ad191c7f8645848e7291df62e",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "bdc927c67b2a793b4383103d0c4af9dbc4c8535d943e8217084403f24c1b87d0",
@@ -191,7 +191,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Amidar-v3": {
+  "Amidar-v4": {
     "actions": "b199b81b77e4e2a8aad9a5663315bd9f7a65ba9ad191c7f8645848e7291df62e",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -203,7 +203,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "AmidarDeterministic-v3": {
+  "AmidarDeterministic-v4": {
     "actions": "b199b81b77e4e2a8aad9a5663315bd9f7a65ba9ad191c7f8645848e7291df62e",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -215,7 +215,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "AmidarNoFrameskip-v3": {
+  "AmidarNoFrameskip-v4": {
     "actions": "b199b81b77e4e2a8aad9a5663315bd9f7a65ba9ad191c7f8645848e7291df62e",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -227,7 +227,7 @@
     "observations": "441816ae1822e3712143a72cf1c0fc99b0f965ecfe90db8416d223f185cd8a91",
     "rewards": "c47f4f01118ce98d19766582338336b83632828140ffc6bef23fede5be614ac1"
   },
-  "Assault-ram-v3": {
+  "Assault-ram-v4": {
     "actions": "d8701aff9cdc2b141b4766483c2221e701c3e1e0e7ba94be54a005402022bc92",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "7e9cfe568e66c60a729f2725d301997f3671a968b4eea291fa34d289bffcec1b",
@@ -239,7 +239,7 @@
     "observations": "ac2d31e6133138231a3e21f704489ed3e3ef9afba170c5ecdd506bdecbac49a4",
     "rewards": "322555e5ba41a7baad194277633fcb90092e1757b5eb1c2e9ea5a171edb34889"
   },
-  "Assault-ramDeterministic-v3": {
+  "Assault-ramDeterministic-v4": {
     "actions": "d8701aff9cdc2b141b4766483c2221e701c3e1e0e7ba94be54a005402022bc92",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "6beb404b4a52dabf9aa4925a205b1eda0a0258089ec79ebf9cce502a060cf959",
@@ -251,7 +251,7 @@
     "observations": "47956751aa2179ad483456cb4c6651fae792a51f26ec4073d93f4417af00eddb",
     "rewards": "d64337051abec22cb81f325e0d1781e14c78b2d63a6f4bba3be36500e33bae63"
   },
-  "Assault-ramNoFrameskip-v3": {
+  "Assault-ramNoFrameskip-v4": {
     "actions": "d8701aff9cdc2b141b4766483c2221e701c3e1e0e7ba94be54a005402022bc92",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "3026000ac2ff7d655ad0a4d246a2678c3db2fd267f8f18c42648673f3188b9c2",
@@ -263,7 +263,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "c47f4f01118ce98d19766582338336b83632828140ffc6bef23fede5be614ac1"
   },
-  "Assault-v3": {
+  "Assault-v4": {
     "actions": "d8701aff9cdc2b141b4766483c2221e701c3e1e0e7ba94be54a005402022bc92",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -275,7 +275,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "322555e5ba41a7baad194277633fcb90092e1757b5eb1c2e9ea5a171edb34889"
   },
-  "AssaultDeterministic-v3": {
+  "AssaultDeterministic-v4": {
     "actions": "d8701aff9cdc2b141b4766483c2221e701c3e1e0e7ba94be54a005402022bc92",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -287,7 +287,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "d64337051abec22cb81f325e0d1781e14c78b2d63a6f4bba3be36500e33bae63"
   },
-  "AssaultNoFrameskip-v3": {
+  "AssaultNoFrameskip-v4": {
     "actions": "d8701aff9cdc2b141b4766483c2221e701c3e1e0e7ba94be54a005402022bc92",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -299,7 +299,7 @@
     "observations": "26c19bb391d30853a7ba154eda46b350cc1736abd573d8810f5088456f283398",
     "rewards": "c4e329852a49d5f998a0684a484851da07fd8a6194a77c0d5d52d0f4d4a9acea"
   },
-  "Asterix-ram-v3": {
+  "Asterix-ram-v4": {
     "actions": "680dc83e85ea9c0ec0bed4ba7ae3a87dbf66cc40db1922a0ec9debfca671766f",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "3685ab4daedcbf1356386e2cb28fc3cee6b666073c57ef53916a217c638f3c61",
@@ -311,7 +311,7 @@
     "observations": "cb96a598538473c97206f708f8ae77996b1b42f3a2260e37fc116d088cedf0b4",
     "rewards": "ee45fb16e16b28c943824a77a2152c9cb03ef45991ccbe7b079622ae6e993a74"
   },
-  "Asterix-ramDeterministic-v3": {
+  "Asterix-ramDeterministic-v4": {
     "actions": "680dc83e85ea9c0ec0bed4ba7ae3a87dbf66cc40db1922a0ec9debfca671766f",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "231c370b97fa47238e3161ae57abff1332c43a9bc4d6aeda54fcbe7afdc8c921",
@@ -323,7 +323,7 @@
     "observations": "e5786eff4a1822ca463d327d0efffd0bde912100db05cac53767eb5fecfe69a2",
     "rewards": "3193b841f5f8736abbc624d9e69efb63f272cf8a22b0fab78b0f39391e02ec00"
   },
-  "Asterix-ramNoFrameskip-v3": {
+  "Asterix-ramNoFrameskip-v4": {
     "actions": "680dc83e85ea9c0ec0bed4ba7ae3a87dbf66cc40db1922a0ec9debfca671766f",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "9fe7a1c89dd81f5d595025a461f35d02a789f98f59b103cd6cd521dfa37aee7d",
@@ -335,7 +335,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "c4e329852a49d5f998a0684a484851da07fd8a6194a77c0d5d52d0f4d4a9acea"
   },
-  "Asterix-v3": {
+  "Asterix-v4": {
     "actions": "680dc83e85ea9c0ec0bed4ba7ae3a87dbf66cc40db1922a0ec9debfca671766f",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -347,7 +347,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "ee45fb16e16b28c943824a77a2152c9cb03ef45991ccbe7b079622ae6e993a74"
   },
-  "AsterixDeterministic-v3": {
+  "AsterixDeterministic-v4": {
     "actions": "680dc83e85ea9c0ec0bed4ba7ae3a87dbf66cc40db1922a0ec9debfca671766f",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -359,7 +359,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "3193b841f5f8736abbc624d9e69efb63f272cf8a22b0fab78b0f39391e02ec00"
   },
-  "AsterixNoFrameskip-v3": {
+  "AsterixNoFrameskip-v4": {
     "actions": "680dc83e85ea9c0ec0bed4ba7ae3a87dbf66cc40db1922a0ec9debfca671766f",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -371,7 +371,7 @@
     "observations": "5c6a5bac73d9a0621bd8ad32c22f85c5f00f410c5944939d30dc156182cd59de",
     "rewards": "2f249f97d59961bde3cafe1dc0e6d9689fda559387caa6c3711e0a26f01673a7"
   },
-  "Asteroids-ram-v3": {
+  "Asteroids-ram-v4": {
     "actions": "ae43ac06914f7dab6de7889e1f7b99a91aa10f0204e012bd95e21e929ceda91d",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "5a3ed38060c84276132caebd7ffe40fda2cfbb83179a8ff90f716d4b146049b0",
@@ -383,7 +383,7 @@
     "observations": "c743dedde3a44130022d790999f136010370940fb9902dd82abe8c8842fe9610",
     "rewards": "8f81de17b9368de3a2d3c7d1e0f80660dd873a2d14eed5c96db8b392eb613b09"
   },
-  "Asteroids-ramDeterministic-v3": {
+  "Asteroids-ramDeterministic-v4": {
     "actions": "ae43ac06914f7dab6de7889e1f7b99a91aa10f0204e012bd95e21e929ceda91d",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "c743dedde3a44130022d790999f136010370940fb9902dd82abe8c8842fe9610",
@@ -395,7 +395,7 @@
     "observations": "3349dc899508a15af78b60fac36f8cd39e86589e7e8e4a0ecbe7db483718757b",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Asteroids-ramNoFrameskip-v3": {
+  "Asteroids-ramNoFrameskip-v4": {
     "actions": "ae43ac06914f7dab6de7889e1f7b99a91aa10f0204e012bd95e21e929ceda91d",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "ff798c44c183080086e4487d1064b7a7f778fba7caef354ac6dafff71a3e2cd6",
@@ -407,7 +407,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "2f249f97d59961bde3cafe1dc0e6d9689fda559387caa6c3711e0a26f01673a7"
   },
-  "Asteroids-v3": {
+  "Asteroids-v4": {
     "actions": "ae43ac06914f7dab6de7889e1f7b99a91aa10f0204e012bd95e21e929ceda91d",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -419,7 +419,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "8f81de17b9368de3a2d3c7d1e0f80660dd873a2d14eed5c96db8b392eb613b09"
   },
-  "AsteroidsDeterministic-v3": {
+  "AsteroidsDeterministic-v4": {
     "actions": "ae43ac06914f7dab6de7889e1f7b99a91aa10f0204e012bd95e21e929ceda91d",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -431,7 +431,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "AsteroidsNoFrameskip-v3": {
+  "AsteroidsNoFrameskip-v4": {
     "actions": "ae43ac06914f7dab6de7889e1f7b99a91aa10f0204e012bd95e21e929ceda91d",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -443,7 +443,7 @@
     "observations": "08f6fc921ac79a6a7efe402b5e3a4b72bbf80a92cb22d2b70050c8d122326ccf",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Atlantis-ram-v3": {
+  "Atlantis-ram-v4": {
     "actions": "ec9b2f7d83b6591999b67843d51ac0947dd5602d6c89b02b2f4614d36e7f6513",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "bdb7d3413731d52a862ddbe78d5ab07567ff339b6a928f770cf3845d5a69fc49",
@@ -455,7 +455,7 @@
     "observations": "3bcaa916493dbf149352366f9897ff05dc3ea847031115412e22fdab1b45d1eb",
     "rewards": "92c93cb2640f5c91fbe282ae94b567fdb79bf002a6d3ad0430310651c0144194"
   },
-  "Atlantis-ramDeterministic-v3": {
+  "Atlantis-ramDeterministic-v4": {
     "actions": "ec9b2f7d83b6591999b67843d51ac0947dd5602d6c89b02b2f4614d36e7f6513",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "2731524ad33acc177b0378396fc353ca2fea9487d94926092f786deea71d3557",
@@ -467,7 +467,7 @@
     "observations": "242e7eef1488a3418c86d20a3c638dce1ec6fb504ef9437d5c2595b64499b0ed",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Atlantis-ramNoFrameskip-v3": {
+  "Atlantis-ramNoFrameskip-v4": {
     "actions": "ec9b2f7d83b6591999b67843d51ac0947dd5602d6c89b02b2f4614d36e7f6513",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "389cd145f9eb0c72f17e6eccbd61500c6df27becce58e5a151527a9ed6ea21aa",
@@ -479,7 +479,7 @@
     "observations": "dc6a89cebe2307516a293b41439499bc899adeca63abddd0ebd36b042355bafb",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Atlantis-v3": {
+  "Atlantis-v4": {
     "actions": "ec9b2f7d83b6591999b67843d51ac0947dd5602d6c89b02b2f4614d36e7f6513",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "dc6a89cebe2307516a293b41439499bc899adeca63abddd0ebd36b042355bafb",
@@ -491,7 +491,7 @@
     "observations": "dc6a89cebe2307516a293b41439499bc899adeca63abddd0ebd36b042355bafb",
     "rewards": "92c93cb2640f5c91fbe282ae94b567fdb79bf002a6d3ad0430310651c0144194"
   },
-  "AtlantisDeterministic-v3": {
+  "AtlantisDeterministic-v4": {
     "actions": "ec9b2f7d83b6591999b67843d51ac0947dd5602d6c89b02b2f4614d36e7f6513",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "dc6a89cebe2307516a293b41439499bc899adeca63abddd0ebd36b042355bafb",
@@ -503,7 +503,7 @@
     "observations": "dc6a89cebe2307516a293b41439499bc899adeca63abddd0ebd36b042355bafb",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "AtlantisNoFrameskip-v3": {
+  "AtlantisNoFrameskip-v4": {
     "actions": "ec9b2f7d83b6591999b67843d51ac0947dd5602d6c89b02b2f4614d36e7f6513",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "dc6a89cebe2307516a293b41439499bc899adeca63abddd0ebd36b042355bafb",
@@ -515,7 +515,7 @@
     "observations": "f89dfc3bfee10306dadb3b5367006b8921ef4e6575cb36788470b0d491299ed2",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "BankHeist-ram-v3": {
+  "BankHeist-ram-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "b4e79843660cdc108eec06e085848cd3119f4633509e65f9d83dc9e233d4f6f8",
@@ -527,7 +527,7 @@
     "observations": "fbd213be5126774896f9f034354f91080080c655551907d17c499ddbbdc6a354",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "BankHeist-ramDeterministic-v3": {
+  "BankHeist-ramDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "5dfaa9061d05325609bdcc866a66752bf8e5f8ceff32760c119d06f840676928",
@@ -539,7 +539,7 @@
     "observations": "c61fdad6600ed12c7b8509c1fc1d3f179907be8811e9d8a4c323e7f7509479f2",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "BankHeist-ramNoFrameskip-v3": {
+  "BankHeist-ramNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "17d7a2014692ec15b3a15d473ed379caeac86a397055b7cdc4a39624dbe1a78e",
@@ -551,7 +551,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "BankHeist-v3": {
+  "BankHeist-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -563,7 +563,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "BankHeistDeterministic-v3": {
+  "BankHeistDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -575,7 +575,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "BankHeistNoFrameskip-v3": {
+  "BankHeistNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -587,7 +587,7 @@
     "observations": "ee35be196a169269215a2c1e836eaeb85ac9cac0f7ede4d7055373d928f950df",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "BattleZone-ram-v3": {
+  "BattleZone-ram-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "19e12e9306060256294783816958db615bf2e68d2357cdaa706802f2c33b2bf2",
@@ -599,7 +599,7 @@
     "observations": "9207d4c0fb9d5befe2ff17edfd4039bcb54963d57f934393e67c87b6755b4b80",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "BattleZone-ramDeterministic-v3": {
+  "BattleZone-ramDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "cb8e59dc592cbe90f2449daf02b604608379af923d3011d49bd68f8e80e3ee8e",
@@ -611,7 +611,7 @@
     "observations": "d4147346586693e3e91178e244b69621b33fe96ebe0b0de6997eca84d4f12df0",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "BattleZone-ramNoFrameskip-v3": {
+  "BattleZone-ramNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "91ee8f4303547effe461b2d326bb9024858811413af7d6e1aca9876824cf50bf",
@@ -623,7 +623,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "BattleZone-v3": {
+  "BattleZone-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -635,7 +635,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "BattleZoneDeterministic-v3": {
+  "BattleZoneDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -647,7 +647,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "BattleZoneNoFrameskip-v3": {
+  "BattleZoneNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -659,7 +659,7 @@
     "observations": "2273d1a6753f8f09006b183caf7595c00dbba5e1c21a669b4d34ab401f378039",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "BeamRider-ram-v3": {
+  "BeamRider-ram-v4": {
     "actions": "680dc83e85ea9c0ec0bed4ba7ae3a87dbf66cc40db1922a0ec9debfca671766f",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "2273d1a6753f8f09006b183caf7595c00dbba5e1c21a669b4d34ab401f378039",
@@ -671,7 +671,7 @@
     "observations": "5bb6eb6782e092e88c9256ce2edc73908eb5783b72d158c7b7e731a63c0e9b65",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "BeamRider-ramDeterministic-v3": {
+  "BeamRider-ramDeterministic-v4": {
     "actions": "680dc83e85ea9c0ec0bed4ba7ae3a87dbf66cc40db1922a0ec9debfca671766f",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "5bb6eb6782e092e88c9256ce2edc73908eb5783b72d158c7b7e731a63c0e9b65",
@@ -683,7 +683,7 @@
     "observations": "6b9e50dd916c312cd7aeec13ff28f196ae40c0117030cb6fb8eb4d84c9d89a7b",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "BeamRider-ramNoFrameskip-v3": {
+  "BeamRider-ramNoFrameskip-v4": {
     "actions": "680dc83e85ea9c0ec0bed4ba7ae3a87dbf66cc40db1922a0ec9debfca671766f",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "6b9e50dd916c312cd7aeec13ff28f196ae40c0117030cb6fb8eb4d84c9d89a7b",
@@ -695,7 +695,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "BeamRider-v3": {
+  "BeamRider-v4": {
     "actions": "680dc83e85ea9c0ec0bed4ba7ae3a87dbf66cc40db1922a0ec9debfca671766f",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -707,7 +707,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "BeamRiderDeterministic-v3": {
+  "BeamRiderDeterministic-v4": {
     "actions": "680dc83e85ea9c0ec0bed4ba7ae3a87dbf66cc40db1922a0ec9debfca671766f",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -719,7 +719,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "BeamRiderNoFrameskip-v3": {
+  "BeamRiderNoFrameskip-v4": {
     "actions": "680dc83e85ea9c0ec0bed4ba7ae3a87dbf66cc40db1922a0ec9debfca671766f",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -731,7 +731,7 @@
     "observations": "57fa0390f59d084e2643f0fbf6972d3c73e9688850a831744481dff4c2c5994b",
     "rewards": "ff55d98017e6d99ce3bcc60fc0a658f7c27f0b934285eb3e2fe26cb700d5560d"
   },
-  "Berzerk-ram-v3": {
+  "Berzerk-ram-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "8982673ce1376b9892703f88075e458ab1313859d383613d5733c22f11a8df24",
@@ -743,7 +743,7 @@
     "observations": "a623d728c732a87737865644141de016eb1cd503aedaeeb0e55126ee8aec40fc",
     "rewards": "fdfa0d3d504e6e7a5848bf6abf2dd810261d527ec47de700c7e50b41044d0d22"
   },
-  "Berzerk-ramDeterministic-v3": {
+  "Berzerk-ramDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "242cd2df736bb3dd972bc459dfb0bd2a8a44dba02d741e1ac989bcb3160e10cd",
@@ -755,7 +755,7 @@
     "observations": "ed4feed5fdcdc1969fc3ca18edc339ba5c0f917815036b99fe688a786fda00cc",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Berzerk-ramNoFrameskip-v3": {
+  "Berzerk-ramNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "0f909b0bb5bf78d6672202723add7fbf67d4c654194b5f7617d940f8e1a82539",
@@ -767,7 +767,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "ff55d98017e6d99ce3bcc60fc0a658f7c27f0b934285eb3e2fe26cb700d5560d"
   },
-  "Berzerk-v3": {
+  "Berzerk-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -779,7 +779,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "fdfa0d3d504e6e7a5848bf6abf2dd810261d527ec47de700c7e50b41044d0d22"
   },
-  "BerzerkDeterministic-v3": {
+  "BerzerkDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -791,7 +791,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "BerzerkNoFrameskip-v3": {
+  "BerzerkNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -809,7 +809,7 @@
     "observations": "c096c24952ed13b33f9c046404db6f307138d49f382f426443c6730cc1b871ca",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Bowling-ram-v3": {
+  "Bowling-ram-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "b07706dfd8365ca52ff72735e60e383fc18b158cc8bfb0694ec78ae4ae17c4e4",
@@ -821,7 +821,7 @@
     "observations": "ec5fe2791a15bdbf92249723a2524a2a718da104c9612a38bfe20c3d151a23ff",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Bowling-ramDeterministic-v3": {
+  "Bowling-ramDeterministic-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "ec5fe2791a15bdbf92249723a2524a2a718da104c9612a38bfe20c3d151a23ff",
@@ -833,7 +833,7 @@
     "observations": "3bbbf29a40f09b9ff6f63b84c1d3ee4b1983d68f4dd9738e581d5716182c3d8f",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Bowling-ramNoFrameskip-v3": {
+  "Bowling-ramNoFrameskip-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "4f7478eae716b092ea8c73a3398e85ab5ae478ab0bc2bf6ed8b3f927bc753475",
@@ -845,7 +845,7 @@
     "observations": "4a78d482e9cffd0d06088ac36311962a5fea18a223bd670c1bc364b0e1aa7715",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Bowling-v3": {
+  "Bowling-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "4a78d482e9cffd0d06088ac36311962a5fea18a223bd670c1bc364b0e1aa7715",
@@ -857,7 +857,7 @@
     "observations": "4a78d482e9cffd0d06088ac36311962a5fea18a223bd670c1bc364b0e1aa7715",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "BowlingDeterministic-v3": {
+  "BowlingDeterministic-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "4a78d482e9cffd0d06088ac36311962a5fea18a223bd670c1bc364b0e1aa7715",
@@ -869,7 +869,7 @@
     "observations": "4a78d482e9cffd0d06088ac36311962a5fea18a223bd670c1bc364b0e1aa7715",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "BowlingNoFrameskip-v3": {
+  "BowlingNoFrameskip-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "4a78d482e9cffd0d06088ac36311962a5fea18a223bd670c1bc364b0e1aa7715",
@@ -881,7 +881,7 @@
     "observations": "974d2f7a6eb58819ad0556d4108145c1991c88103435f2393a316f7886134998",
     "rewards": "e15267f10aff441ed9afdf25dc1b6b5f7813a4e97c4f3817bd2d757a3c913c4e"
   },
-  "Boxing-ram-v3": {
+  "Boxing-ram-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "9fc0ed6e358ab69ac458dadaa53741389ac8e19c6bdfc68932c96aea50b4b404",
@@ -893,7 +893,7 @@
     "observations": "458ec3e46587f8e78edcfbf9761e99b5135add9625722d4b67f4594bd56239ce",
     "rewards": "b774641193c72918024c4c7d9997ba3a9b4be04c4de421622a01f6c06a959529"
   },
-  "Boxing-ramDeterministic-v3": {
+  "Boxing-ramDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "89661865bd48e4bbf582c9dc5db906e9f2f47185544ba959dba68f88336a617e",
@@ -905,7 +905,7 @@
     "observations": "46244a1d9dad7b255bd8746e16d46ccc72c6e0d11c863d7c55d6eba1672a0f4c",
     "rewards": "d302e626f3bedbbb3d228652f45f43dc494b40d3cc6bc94276becfd400f676e8"
   },
-  "Boxing-ramNoFrameskip-v3": {
+  "Boxing-ramNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "5713d2eff9a05b4cefb03e3835a29890c1b45e935de2825bc64e802a10847a0d",
@@ -917,7 +917,7 @@
     "observations": "384a78298e55047fba47a5f3311ef54a7fc8557afcf9696f2aa50019b1528d2a",
     "rewards": "e15267f10aff441ed9afdf25dc1b6b5f7813a4e97c4f3817bd2d757a3c913c4e"
   },
-  "Boxing-v3": {
+  "Boxing-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "384a78298e55047fba47a5f3311ef54a7fc8557afcf9696f2aa50019b1528d2a",
@@ -929,7 +929,7 @@
     "observations": "384a78298e55047fba47a5f3311ef54a7fc8557afcf9696f2aa50019b1528d2a",
     "rewards": "b774641193c72918024c4c7d9997ba3a9b4be04c4de421622a01f6c06a959529"
   },
-  "BoxingDeterministic-v3": {
+  "BoxingDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "384a78298e55047fba47a5f3311ef54a7fc8557afcf9696f2aa50019b1528d2a",
@@ -941,7 +941,7 @@
     "observations": "384a78298e55047fba47a5f3311ef54a7fc8557afcf9696f2aa50019b1528d2a",
     "rewards": "d302e626f3bedbbb3d228652f45f43dc494b40d3cc6bc94276becfd400f676e8"
   },
-  "BoxingNoFrameskip-v3": {
+  "BoxingNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "384a78298e55047fba47a5f3311ef54a7fc8557afcf9696f2aa50019b1528d2a",
@@ -953,7 +953,7 @@
     "observations": "f778da9750269f07209ca7fae9e5c493abc38793f4539cc7948c684e1cc32056",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Breakout-ram-v3": {
+  "Breakout-ram-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "a9fe8274ee82506b3d5ee3dd60c895dd2b04cf6c3e1eaaeef43e529e349e67e5",
@@ -965,7 +965,7 @@
     "observations": "a0d12f5e78b84566a780b0970f675c90d056f0598c2c9f7542501d56ccef1a5d",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Breakout-ramDeterministic-v3": {
+  "Breakout-ramDeterministic-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "959eb57322256d4bdb6942b47582b68a892ab8d56e08ced9f904e85789b876b7",
@@ -977,7 +977,7 @@
     "observations": "f16f87dff2929881be147319aa36960e26c837e9e10d722990eb578a91572d0f",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Breakout-ramNoFrameskip-v3": {
+  "Breakout-ramNoFrameskip-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "1547a2f6a7c2de7e31c57b256ec385ac95d56eba27d67901763526bee2de821c",
@@ -989,7 +989,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Breakout-v3": {
+  "Breakout-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -1001,7 +1001,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "BreakoutDeterministic-v3": {
+  "BreakoutDeterministic-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -1013,7 +1013,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "BreakoutNoFrameskip-v3": {
+  "BreakoutNoFrameskip-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -1025,7 +1025,7 @@
     "observations": "9e050e6bfe937684b9562cb1b7e9f30d07c22f92094854701c4b4d4a6e2aff22",
     "rewards": "9c7f3d29849c6aa7791f3e8b0c48c882ad6bdb0ceef716b86e3a9dea3989826b"
   },
-  "Carnival-ram-v3": {
+  "Carnival-ram-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "6300fadf919d7bfea0231b90b89b6ffc5879943d3c2c76deb762b06d1b2c560d",
@@ -1037,7 +1037,7 @@
     "observations": "166f1384782d38f6c71fd45f10797b6954dab2596d02e5354da9fd8ea67b4eb0",
     "rewards": "b35136a2aed637c1c765d189de4eb2e1b56e5280222742a0ff66088edaba49b8"
   },
-  "Carnival-ramDeterministic-v3": {
+  "Carnival-ramDeterministic-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "ceefcda5f052c006b6d16cbfea8b4b860b1d9da4daafd2e0eafc5f8485a12928",
@@ -1049,7 +1049,7 @@
     "observations": "07aac30ac99b650c92f8ea7b7b075056f6c08002cd4384e77429e937d9a41f8a",
     "rewards": "03dcc20ce9ace38c38f6c981c1d7140b3631e0be97c237b8c5209fee3d683255"
   },
-  "Carnival-ramNoFrameskip-v3": {
+  "Carnival-ramNoFrameskip-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "9e1ecdebdbe1287008686585ee5d559e052823ba49038199ee94ae2953dd687e",
@@ -1061,7 +1061,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "9c7f3d29849c6aa7791f3e8b0c48c882ad6bdb0ceef716b86e3a9dea3989826b"
   },
-  "Carnival-v3": {
+  "Carnival-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -1073,7 +1073,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "b35136a2aed637c1c765d189de4eb2e1b56e5280222742a0ff66088edaba49b8"
   },
-  "CarnivalDeterministic-v3": {
+  "CarnivalDeterministic-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -1085,7 +1085,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "03dcc20ce9ace38c38f6c981c1d7140b3631e0be97c237b8c5209fee3d683255"
   },
-  "CarnivalNoFrameskip-v3": {
+  "CarnivalNoFrameskip-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -1109,7 +1109,7 @@
     "observations": "595a8f82b6874f00fc3490fc855b1c41cf3c563bad52351b014ba3e298f7e471",
     "rewards": "b2e608a577cb1019219f5f81f65bb0da49b9539b7d305f94b78cfba7391d0fcc"
   },
-  "Centipede-ram-v3": {
+  "Centipede-ram-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "83be894543690e509b015db0f8c09c7d07b2acbc8c62c817826a706efe48be67",
@@ -1121,7 +1121,7 @@
     "observations": "1f198342f4d10d2de6b7f5f8fa0c7ba081f9dd3f76331b104808c1cab590ee66",
     "rewards": "f5eb70e0b00b725217c45f72acc3d6ca6786ab72a79689df1fad4fd58abf9f3a"
   },
-  "Centipede-ramDeterministic-v3": {
+  "Centipede-ramDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "900afec9f06ada09910a15ac15f1e2697d573d6faf791d89101e8319b8d604b1",
@@ -1133,7 +1133,7 @@
     "observations": "5226ebd24b57a550912fb8837728353d070d9a5cdbdadd1bf90f36bdd60ef157",
     "rewards": "a0d92a3d892cb9e6501358e7f95f8d1f95db43b690ff29379f9aa1b11bb69d4d"
   },
-  "Centipede-ramNoFrameskip-v3": {
+  "Centipede-ramNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "9d70dd85b9c9234b9bfb3e1982865f7659860d09828b72042195469953caadca",
@@ -1145,7 +1145,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "b2e608a577cb1019219f5f81f65bb0da49b9539b7d305f94b78cfba7391d0fcc"
   },
-  "Centipede-v3": {
+  "Centipede-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -1157,7 +1157,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "f5eb70e0b00b725217c45f72acc3d6ca6786ab72a79689df1fad4fd58abf9f3a"
   },
-  "CentipedeDeterministic-v3": {
+  "CentipedeDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -1169,7 +1169,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "a0d92a3d892cb9e6501358e7f95f8d1f95db43b690ff29379f9aa1b11bb69d4d"
   },
-  "CentipedeNoFrameskip-v3": {
+  "CentipedeNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -1181,7 +1181,7 @@
     "observations": "26c4fc5d42721032d31690c32904c893d8899d2fa8ecd0b601c9d7dd10f40ea2",
     "rewards": "5e0b7b7798e655375f01b6b728750e389373992b8712b6f8f5cd0013352f4151"
   },
-  "ChopperCommand-ram-v3": {
+  "ChopperCommand-ram-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "38dbf22227185f9f4d5975d06cae2ec32766c1a3fa3bd457e32a9b2ae354ef2e",
@@ -1193,7 +1193,7 @@
     "observations": "4d81bbcabc3818e7b6daa59968dc28e79c5e09df25b6193db742ba74b5bb1542",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "ChopperCommand-ramDeterministic-v3": {
+  "ChopperCommand-ramDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "4eb30b37d9ab49bd727ec73531624eca3dbc802f9da5cb16b796a91d56fe9271",
@@ -1205,7 +1205,7 @@
     "observations": "649cc15f60a5d72e02b980835f69eced89ec2324524406a4ef11b19c294d53e2",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "ChopperCommand-ramNoFrameskip-v3": {
+  "ChopperCommand-ramNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "c30ec6fcdb8a829ea8a2658eed85b750f79d32ddf08899a45358308f25490360",
@@ -1217,7 +1217,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "5e0b7b7798e655375f01b6b728750e389373992b8712b6f8f5cd0013352f4151"
   },
-  "ChopperCommand-v3": {
+  "ChopperCommand-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -1229,7 +1229,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "ChopperCommandDeterministic-v3": {
+  "ChopperCommandDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -1241,7 +1241,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "ChopperCommandNoFrameskip-v3": {
+  "ChopperCommandNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -1259,7 +1259,7 @@
     "observations": "a4b3166d896d94a3e6862e48d1543a9acb7e4f705f66821ab32894ef9c225205",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "CrazyClimber-ram-v3": {
+  "CrazyClimber-ram-v4": {
     "actions": "680dc83e85ea9c0ec0bed4ba7ae3a87dbf66cc40db1922a0ec9debfca671766f",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "a4b3166d896d94a3e6862e48d1543a9acb7e4f705f66821ab32894ef9c225205",
@@ -1271,7 +1271,7 @@
     "observations": "11e071d5ce25cd58a4e0b5dfd5389f458c59fb76c4a2db7984e75413281a45d4",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "CrazyClimber-ramDeterministic-v3": {
+  "CrazyClimber-ramDeterministic-v4": {
     "actions": "680dc83e85ea9c0ec0bed4ba7ae3a87dbf66cc40db1922a0ec9debfca671766f",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "93c00f4ab6d5bdd82a174262cec2f96745d492c951290034f05179307f6b115a",
@@ -1283,7 +1283,7 @@
     "observations": "5f9e749923b1848eef85b22a6e026a6435da309c8d3ff6f2df07a50263a7b8c2",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "CrazyClimber-ramNoFrameskip-v3": {
+  "CrazyClimber-ramNoFrameskip-v4": {
     "actions": "680dc83e85ea9c0ec0bed4ba7ae3a87dbf66cc40db1922a0ec9debfca671766f",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "5f9e749923b1848eef85b22a6e026a6435da309c8d3ff6f2df07a50263a7b8c2",
@@ -1295,7 +1295,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "CrazyClimber-v3": {
+  "CrazyClimber-v4": {
     "actions": "680dc83e85ea9c0ec0bed4ba7ae3a87dbf66cc40db1922a0ec9debfca671766f",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -1307,7 +1307,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "CrazyClimberDeterministic-v3": {
+  "CrazyClimberDeterministic-v4": {
     "actions": "680dc83e85ea9c0ec0bed4ba7ae3a87dbf66cc40db1922a0ec9debfca671766f",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -1319,7 +1319,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "CrazyClimberNoFrameskip-v3": {
+  "CrazyClimberNoFrameskip-v4": {
     "actions": "680dc83e85ea9c0ec0bed4ba7ae3a87dbf66cc40db1922a0ec9debfca671766f",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -1331,7 +1331,7 @@
     "observations": "b713ac25b8fd48cc904bccc95276a6be381f42aa1b07f1e929ae2a7e38361cc3",
     "rewards": "b4b3ebedaf5634e5b8a4f3e0ed91114096023a092a6f3fcf60b52dcd012930c3"
   },
-  "DemonAttack-ram-v3": {
+  "DemonAttack-ram-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "0330f0a65e8557ffd97a57dfaaab3df0c9ea19a5600305786d0e843531c2cbd7",
@@ -1343,7 +1343,7 @@
     "observations": "b3b01f2a719f5407a0f1f7b89502ae2f2992a79196f036410085801571c58c24",
     "rewards": "2d21ad6301c542270250c7535f5094b7a86185e9069062ad3ae8cf096cedb627"
   },
-  "DemonAttack-ramDeterministic-v3": {
+  "DemonAttack-ramDeterministic-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "cead1885e06c8bdfe4f5879d134f5031201c776015a182f018c839c7071bbefd",
@@ -1355,7 +1355,7 @@
     "observations": "873b13d5918c462e258e18ac043a85fce704d51d2094628997eab6a9929d3bfc",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "DemonAttack-ramNoFrameskip-v3": {
+  "DemonAttack-ramNoFrameskip-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "80da6b73c0c64082c817922005bf501d4357c75acb49fe785679ad228bdf1f4a",
@@ -1367,7 +1367,7 @@
     "observations": "cf2774f2ec508c0c53af558f71510961386b5f37b30a37aa40d407434687b0d3",
     "rewards": "b4b3ebedaf5634e5b8a4f3e0ed91114096023a092a6f3fcf60b52dcd012930c3"
   },
-  "DemonAttack-v3": {
+  "DemonAttack-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "cf2774f2ec508c0c53af558f71510961386b5f37b30a37aa40d407434687b0d3",
@@ -1379,7 +1379,7 @@
     "observations": "cf2774f2ec508c0c53af558f71510961386b5f37b30a37aa40d407434687b0d3",
     "rewards": "2d21ad6301c542270250c7535f5094b7a86185e9069062ad3ae8cf096cedb627"
   },
-  "DemonAttackDeterministic-v3": {
+  "DemonAttackDeterministic-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "cf2774f2ec508c0c53af558f71510961386b5f37b30a37aa40d407434687b0d3",
@@ -1391,7 +1391,7 @@
     "observations": "cf2774f2ec508c0c53af558f71510961386b5f37b30a37aa40d407434687b0d3",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "DemonAttackNoFrameskip-v3": {
+  "DemonAttackNoFrameskip-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "cf2774f2ec508c0c53af558f71510961386b5f37b30a37aa40d407434687b0d3",
@@ -1403,7 +1403,7 @@
     "observations": "13d54b74b66dc16879ce2416cf5c71b2ab99e5af5bc21c4471d95445d9ce8c67",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "DoubleDunk-ram-v3": {
+  "DoubleDunk-ram-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "ff7e614e54ee280b33ee0f8ccc21b92b89c90bcdcf3b952ce25d08f47c0631d9",
@@ -1415,7 +1415,7 @@
     "observations": "2155d5d2d2d61841af0517adc6f349b13072b54232be44326240ab0a08919373",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "DoubleDunk-ramDeterministic-v3": {
+  "DoubleDunk-ramDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "6d6f084784368efbd11c64516ee09e1913ba86ffc4880fd0e4a003892465e382",
@@ -1427,7 +1427,7 @@
     "observations": "d643abcd791acb6f930cab89c339b49f22c6fcfc0bbea15168edf71fa27cbad7",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "DoubleDunk-ramNoFrameskip-v3": {
+  "DoubleDunk-ramNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "1df62519f113a8bf8f994bd3272381d53d86a2add5b1f01adc359a2d44456cac",
@@ -1439,7 +1439,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "DoubleDunk-v3": {
+  "DoubleDunk-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -1451,7 +1451,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "DoubleDunkDeterministic-v3": {
+  "DoubleDunkDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -1463,7 +1463,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "DoubleDunkNoFrameskip-v3": {
+  "DoubleDunkNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -1481,7 +1481,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "ElevatorAction-v3": {
+  "ElevatorAction-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -1493,7 +1493,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "ElevatorActionDeterministic-v3": {
+  "ElevatorActionDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -1505,7 +1505,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "ElevatorActionNoFrameskip-v3": {
+  "ElevatorActionNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -1517,7 +1517,7 @@
     "observations": "225be0ab940d420ad37b993fcbe0c8dc798df6ace205bdaf9fb7ec302f79bb6f",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Enduro-ram-v3": {
+  "Enduro-ram-v4": {
     "actions": "680dc83e85ea9c0ec0bed4ba7ae3a87dbf66cc40db1922a0ec9debfca671766f",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "d42f52ec786a3e6fef853149ffb970a3eb8cc8f8141927f20ed2ce729b375fd4",
@@ -1529,7 +1529,7 @@
     "observations": "bc270e18cc26a3cd59c8b22b65e7aa14e24c27eaf86e632a14720d3a32a73ea5",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Enduro-ramDeterministic-v3": {
+  "Enduro-ramDeterministic-v4": {
     "actions": "680dc83e85ea9c0ec0bed4ba7ae3a87dbf66cc40db1922a0ec9debfca671766f",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "61ea08f8cd9a16fae1ec7aa22b603a49059f844cf7a7f6a461c1af4cd9196e2c",
@@ -1541,7 +1541,7 @@
     "observations": "475b439a873dd0bfff983249473adc62e28978b9d1e14bcb94530a9dc54addec",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Enduro-ramNoFrameskip-v3": {
+  "Enduro-ramNoFrameskip-v4": {
     "actions": "680dc83e85ea9c0ec0bed4ba7ae3a87dbf66cc40db1922a0ec9debfca671766f",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "5e5c67a9104adf9934d67b36f20d598fc72e4fc0e3c0c931ee71f5977d57cf46",
@@ -1553,7 +1553,7 @@
     "observations": "2111ec74cebf57f86b3284d1f70a4c8f311b487bac3d9627803288870bcb06eb",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Enduro-v3": {
+  "Enduro-v4": {
     "actions": "680dc83e85ea9c0ec0bed4ba7ae3a87dbf66cc40db1922a0ec9debfca671766f",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "2111ec74cebf57f86b3284d1f70a4c8f311b487bac3d9627803288870bcb06eb",
@@ -1565,7 +1565,7 @@
     "observations": "2111ec74cebf57f86b3284d1f70a4c8f311b487bac3d9627803288870bcb06eb",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "EnduroDeterministic-v3": {
+  "EnduroDeterministic-v4": {
     "actions": "680dc83e85ea9c0ec0bed4ba7ae3a87dbf66cc40db1922a0ec9debfca671766f",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "2111ec74cebf57f86b3284d1f70a4c8f311b487bac3d9627803288870bcb06eb",
@@ -1577,7 +1577,7 @@
     "observations": "2111ec74cebf57f86b3284d1f70a4c8f311b487bac3d9627803288870bcb06eb",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "EnduroNoFrameskip-v3": {
+  "EnduroNoFrameskip-v4": {
     "actions": "680dc83e85ea9c0ec0bed4ba7ae3a87dbf66cc40db1922a0ec9debfca671766f",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "2111ec74cebf57f86b3284d1f70a4c8f311b487bac3d9627803288870bcb06eb",
@@ -1589,7 +1589,7 @@
     "observations": "4a0aad3c8106f278b5876aa745e926b93101c98e840d032eec3ddf231256b601",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "FishingDerby-ram-v3": {
+  "FishingDerby-ram-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "22201b3594fdb569071a5dfbbfcffc96d3e2c19a09d62711e7a8a95c13203607",
@@ -1601,7 +1601,7 @@
     "observations": "5ea65fe74670a2bcdaf7f161e312ecfff0058587e3af2c0089a5946670c3fceb",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "FishingDerby-ramDeterministic-v3": {
+  "FishingDerby-ramDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "e1a457cfd2a9da2cd18a7acb2d4a670afbcae4f4b7932da23d6e401a55004379",
@@ -1613,7 +1613,7 @@
     "observations": "0f4e80157fd2d4b1685bbb2d48b9c1088819a8cc9395182a21d9616d18df3022",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "FishingDerby-ramNoFrameskip-v3": {
+  "FishingDerby-ramNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "256a23d3269fed5daadbb3d41fbe6e52113d9dd24c877d3d003732cda7de1164",
@@ -1625,7 +1625,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "FishingDerby-v3": {
+  "FishingDerby-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -1637,7 +1637,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "FishingDerbyDeterministic-v3": {
+  "FishingDerbyDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -1649,7 +1649,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "FishingDerbyNoFrameskip-v3": {
+  "FishingDerbyNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -1661,7 +1661,7 @@
     "observations": "fa119df64dbe9fc363f8bfadf8eb8bebeaa5a98adb65cc9a37029442ee6011aa",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Freeway-ram-v3": {
+  "Freeway-ram-v4": {
     "actions": "5138748c3c039a57ee365473ef13e5b99329e75a4f71459cd1a0d7919fd6e97b",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "b00b692957790fba85076914547dfb95913316b4934a50382dfb2834025d6c25",
@@ -1673,7 +1673,7 @@
     "observations": "22e769f14faa50baaca159b285f6874d875a0ab7ddd0315cfc789228ad682a23",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Freeway-ramDeterministic-v3": {
+  "Freeway-ramDeterministic-v4": {
     "actions": "5138748c3c039a57ee365473ef13e5b99329e75a4f71459cd1a0d7919fd6e97b",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "a4f62ee61ca967cd67a8bf1de79a5ed8c9a87619ecbbcf90154bc46a0830b87a",
@@ -1685,7 +1685,7 @@
     "observations": "a562ace6c6a26277f3b3b941689623f69458e837e86eda6c2a7a2ccfc1e57e48",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Freeway-ramNoFrameskip-v3": {
+  "Freeway-ramNoFrameskip-v4": {
     "actions": "5138748c3c039a57ee365473ef13e5b99329e75a4f71459cd1a0d7919fd6e97b",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "44b60072856926415b6fa7906cc06ebde463bca2343d41d8797ec95f122767f3",
@@ -1697,7 +1697,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Freeway-v3": {
+  "Freeway-v4": {
     "actions": "5138748c3c039a57ee365473ef13e5b99329e75a4f71459cd1a0d7919fd6e97b",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -1709,7 +1709,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "FreewayDeterministic-v3": {
+  "FreewayDeterministic-v4": {
     "actions": "5138748c3c039a57ee365473ef13e5b99329e75a4f71459cd1a0d7919fd6e97b",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -1721,7 +1721,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "FreewayNoFrameskip-v3": {
+  "FreewayNoFrameskip-v4": {
     "actions": "5138748c3c039a57ee365473ef13e5b99329e75a4f71459cd1a0d7919fd6e97b",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -1733,7 +1733,7 @@
     "observations": "6b288ba3dade7b016085a345975f1bc0f3a8d629c7d6ba7a9905387f93c2385e",
     "rewards": "a09cd4b4d67f256463d4d449440ef1eaeb426a13a66228c343862fd137ca5509"
   },
-  "Frostbite-ram-v3": {
+  "Frostbite-ram-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "6e2eac7228fe3b75f94bdef909666ac37859e97c6b5efebfed6d2709732205c4",
@@ -1745,7 +1745,7 @@
     "observations": "987fc53c593980c33f976ee638f80ce7cc273b21439099ab25ba0072294b4c61",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Frostbite-ramDeterministic-v3": {
+  "Frostbite-ramDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "987fc53c593980c33f976ee638f80ce7cc273b21439099ab25ba0072294b4c61",
@@ -1757,7 +1757,7 @@
     "observations": "28466c6c44877bf7a673fb047b0249282071bc63fee80cf67c5edc08875b6faf",
     "rewards": "725712787f7609c6eb2c15f52f30f622fb596942d2f34aa89ee9406c1d703d70"
   },
-  "Frostbite-ramNoFrameskip-v3": {
+  "Frostbite-ramNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "4f05e9b1076e14e8ea8c21ac122f80b64cae3c76e8fe43a290e1c3bdca94dd37",
@@ -1769,7 +1769,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "a09cd4b4d67f256463d4d449440ef1eaeb426a13a66228c343862fd137ca5509"
   },
-  "Frostbite-v3": {
+  "Frostbite-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -1781,7 +1781,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "FrostbiteDeterministic-v3": {
+  "FrostbiteDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -1793,7 +1793,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "725712787f7609c6eb2c15f52f30f622fb596942d2f34aa89ee9406c1d703d70"
   },
-  "FrostbiteNoFrameskip-v3": {
+  "FrostbiteNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -1817,7 +1817,7 @@
     "observations": "321672b88fd11bc3caf5731298aa0231ede35bfceb0626a61312f85a9d152dc7",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Gopher-ram-v3": {
+  "Gopher-ram-v4": {
     "actions": "f72cb9f7a8c584feab60a4f9ae594cbbb98c472df7d917ebf9a20855bec634ae",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "2fbdbb8533a4196fb9ea12bbed84065bd1cffe59a189b03565e3bd47815fe625",
@@ -1829,7 +1829,7 @@
     "observations": "b1c24e0451ab7897371640adfabb55398cc32d00ef216d47f49f992279a771fa",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Gopher-ramDeterministic-v3": {
+  "Gopher-ramDeterministic-v4": {
     "actions": "f72cb9f7a8c584feab60a4f9ae594cbbb98c472df7d917ebf9a20855bec634ae",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "a027d7a5bf7a13273f78b8bd24086591dffdc692595b690b5e05ea478e49201f",
@@ -1841,7 +1841,7 @@
     "observations": "68993986345a5c69f3ad28ecabaf0436f7f906d2415c2a272b84f51784b42af9",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Gopher-ramNoFrameskip-v3": {
+  "Gopher-ramNoFrameskip-v4": {
     "actions": "f72cb9f7a8c584feab60a4f9ae594cbbb98c472df7d917ebf9a20855bec634ae",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "68993986345a5c69f3ad28ecabaf0436f7f906d2415c2a272b84f51784b42af9",
@@ -1853,7 +1853,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Gopher-v3": {
+  "Gopher-v4": {
     "actions": "f72cb9f7a8c584feab60a4f9ae594cbbb98c472df7d917ebf9a20855bec634ae",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -1865,7 +1865,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "GopherDeterministic-v3": {
+  "GopherDeterministic-v4": {
     "actions": "f72cb9f7a8c584feab60a4f9ae594cbbb98c472df7d917ebf9a20855bec634ae",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -1877,7 +1877,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "GopherNoFrameskip-v3": {
+  "GopherNoFrameskip-v4": {
     "actions": "f72cb9f7a8c584feab60a4f9ae594cbbb98c472df7d917ebf9a20855bec634ae",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -1889,7 +1889,7 @@
     "observations": "05f8717eaec2be27dd4a50eac0c2d6af5432fa819a9f38eed657df769a376958",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Gravitar-ram-v3": {
+  "Gravitar-ram-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "daebb465fec49979e0d698ac6c63958b46b72b9e4ee55eecea03b89604c7d702",
@@ -1901,7 +1901,7 @@
     "observations": "e1a62785708532bd27cf94925b336a8a377ac4f16326a50c90ee92b5cb0f952e",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Gravitar-ramDeterministic-v3": {
+  "Gravitar-ramDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "c06272510e0e444018339bbaa62e3a335a94f558fc73ac053761ad59b032be71",
@@ -1913,7 +1913,7 @@
     "observations": "3299bf1085bebe678498555571f1e269c9b4621c04bc6fd0acc2d7758e8d418b",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Gravitar-ramNoFrameskip-v3": {
+  "Gravitar-ramNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "7ceef28c2a77890a9b62fffdd3d05c72ad78f1ae322c61a2b48942420d1d2d70",
@@ -1925,7 +1925,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Gravitar-v3": {
+  "Gravitar-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -1937,7 +1937,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "GravitarDeterministic-v3": {
+  "GravitarDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -1949,7 +1949,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "GravitarNoFrameskip-v3": {
+  "GravitarNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -1979,7 +1979,7 @@
     "observations": "19d7f1fcf340aa09e46088d7889cd0b563ab8ba1c67b1e209fadd30e3aa91aae",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "IceHockey-ram-v3": {
+  "IceHockey-ram-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "d6a4c0c8b197bf833142b1bda031e7f73e35daaf2aa8a19d17cc1de18d1329f8",
@@ -1991,7 +1991,7 @@
     "observations": "adbc4e7b41998f936566ed79e72dbacc77e0c39847c2db34c8f81613bc3fe571",
     "rewards": "5334039f18872082d4984d449db60f830948783491e3c2d8da63722e92af326d"
   },
-  "IceHockey-ramDeterministic-v3": {
+  "IceHockey-ramDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "9541078f6c54c786ce28592c2d666ddbd3d5a98651efa2d22c44628669140396",
@@ -2003,7 +2003,7 @@
     "observations": "7fb5efdaa4fb512090dc52802a311e46738e01db01c7b59bb38e9427828f66cd",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "IceHockey-ramNoFrameskip-v3": {
+  "IceHockey-ramNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "99bcb6b47c7026df0eebaeddaec00f9352b624327df3879030f1f26e18da9ae1",
@@ -2015,7 +2015,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "IceHockey-v3": {
+  "IceHockey-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -2027,7 +2027,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "5334039f18872082d4984d449db60f830948783491e3c2d8da63722e92af326d"
   },
-  "IceHockeyDeterministic-v3": {
+  "IceHockeyDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -2039,7 +2039,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "IceHockeyNoFrameskip-v3": {
+  "IceHockeyNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -2057,7 +2057,7 @@
     "observations": "7f24f291adec40bb8c6aa099aa3358652bfa47fe68fe17a63cb90e1416a7ff36",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Jamesbond-ram-v3": {
+  "Jamesbond-ram-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "afbfba3362ab3f4111c2d366dada9b76189dc2578cfb6e859cd4f066a190a58f",
@@ -2069,7 +2069,7 @@
     "observations": "cdf4ed47eec1ed175ca19e2521490e7546a78ff5ee711457e92ba0181b49ba79",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Jamesbond-ramDeterministic-v3": {
+  "Jamesbond-ramDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "283a82f2e46a2cfd398dc030a11d7acab93c620362df88b372186d9534d8ea4c",
@@ -2081,7 +2081,7 @@
     "observations": "97b6fa7558e56bbb5b5d26d9bccb4857c4878c48e5d3a55ded678ef67d1eb64c",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Jamesbond-ramNoFrameskip-v3": {
+  "Jamesbond-ramNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "323bb0fc4b9a0250c0d5c7885d6aa0826236d6011cd5f23278e9305c54054bc4",
@@ -2093,7 +2093,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Jamesbond-v3": {
+  "Jamesbond-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -2105,7 +2105,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "JamesbondDeterministic-v3": {
+  "JamesbondDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -2117,7 +2117,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "JamesbondNoFrameskip-v3": {
+  "JamesbondNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -2129,7 +2129,7 @@
     "observations": "d30238a033a869ca6ad19a7daf2c51545e6775ec95965ee05711d23df04ede38",
     "rewards": "6d87cdcb1a5571dfaf8e24f3ad20e9191b0cacb68de2c0d05f512a0480520b5c"
   },
-  "JourneyEscape-ram-v3": {
+  "JourneyEscape-ram-v4": {
     "actions": "60e6f81bb17c1c7cedac4e13370d2c02b176de2ef71fc4f33ae754c42d7b3d0f",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "87cad42d466e3c14855f0cedf41e89c2c1598466b950154d958392f88f25ba98",
@@ -2141,7 +2141,7 @@
     "observations": "925cb2446dff3413d6b27ed6aecc925d4ee31c527a5c8e927d0ed1125d33ee4c",
     "rewards": "5bcc2a9cb589c8dbd5014db4f073918b0cd1a3c101453288ed8bff4dbc769fb9"
   },
-  "JourneyEscape-ramDeterministic-v3": {
+  "JourneyEscape-ramDeterministic-v4": {
     "actions": "60e6f81bb17c1c7cedac4e13370d2c02b176de2ef71fc4f33ae754c42d7b3d0f",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "4897a28b2bbac22df4dfefa333fbf91e7dedf297411fce6bfd451128c7a0119f",
@@ -2153,7 +2153,7 @@
     "observations": "f8db150089281f05e5347b11b0a03961ee8590221a8ed4fcba3793ab460efc47",
     "rewards": "54be3f5680bde52c542e5f49183f330539da91b6e8737675f06f605a167dfd27"
   },
-  "JourneyEscape-ramNoFrameskip-v3": {
+  "JourneyEscape-ramNoFrameskip-v4": {
     "actions": "60e6f81bb17c1c7cedac4e13370d2c02b176de2ef71fc4f33ae754c42d7b3d0f",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "6ee28c1bdfab983269acecebecc05e1dc688fa9c2df43569a16e3c50f2b80e57",
@@ -2165,7 +2165,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "6d87cdcb1a5571dfaf8e24f3ad20e9191b0cacb68de2c0d05f512a0480520b5c"
   },
-  "JourneyEscape-v3": {
+  "JourneyEscape-v4": {
     "actions": "60e6f81bb17c1c7cedac4e13370d2c02b176de2ef71fc4f33ae754c42d7b3d0f",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -2177,7 +2177,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "5bcc2a9cb589c8dbd5014db4f073918b0cd1a3c101453288ed8bff4dbc769fb9"
   },
-  "JourneyEscapeDeterministic-v3": {
+  "JourneyEscapeDeterministic-v4": {
     "actions": "60e6f81bb17c1c7cedac4e13370d2c02b176de2ef71fc4f33ae754c42d7b3d0f",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -2189,7 +2189,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "54be3f5680bde52c542e5f49183f330539da91b6e8737675f06f605a167dfd27"
   },
-  "JourneyEscapeNoFrameskip-v3": {
+  "JourneyEscapeNoFrameskip-v4": {
     "actions": "60e6f81bb17c1c7cedac4e13370d2c02b176de2ef71fc4f33ae754c42d7b3d0f",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -2201,7 +2201,7 @@
     "observations": "1baac426a5b5bd6646d9c26a3428104def0b8810e539c98cefa8641bacafd0f9",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Kangaroo-ram-v3": {
+  "Kangaroo-ram-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "52333e037d63566975452096bb832195511b6fb14441f59f7c540414389d4f77",
@@ -2213,7 +2213,7 @@
     "observations": "686e83656518b15ec9edf900119f6650056ed035f8f6985e0865467fc2ddbdc0",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Kangaroo-ramDeterministic-v3": {
+  "Kangaroo-ramDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "fc756f055a66411daf8fc257451387e997f7f800c9761c291cf4e5d77d75c481",
@@ -2225,7 +2225,7 @@
     "observations": "e4fa5da864721cdd21c499776d2aada44e1736032e84485c7905c697d527ff55",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Kangaroo-ramNoFrameskip-v3": {
+  "Kangaroo-ramNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "e4fa5da864721cdd21c499776d2aada44e1736032e84485c7905c697d527ff55",
@@ -2237,7 +2237,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Kangaroo-v3": {
+  "Kangaroo-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -2249,7 +2249,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "KangarooDeterministic-v3": {
+  "KangarooDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -2261,7 +2261,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "KangarooNoFrameskip-v3": {
+  "KangarooNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -2273,7 +2273,7 @@
     "observations": "a0e6ad4c3e1aab97fc3e670a9a3b69be4469dba2d54614f18821e248d098f073",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Krull-ram-v3": {
+  "Krull-ram-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "d86110eca98569ccaf88a6f93db45bb71b924d240f086c7e07ed111a89e206c0",
@@ -2285,7 +2285,7 @@
     "observations": "7749ca0d9a3d77a5b22ae9fd9920f793f8bc48447fc08bcb2b1c270d4ddd2aae",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Krull-ramDeterministic-v3": {
+  "Krull-ramDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "57a18b37df6fdbefbe75b44f65bac5501b4fb757d50124e490b202295892a19a",
@@ -2297,7 +2297,7 @@
     "observations": "ad8ca03e0544a95d2b78b9984edfa3ac991dca56e3d85a58efb3228753830df6",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Krull-ramNoFrameskip-v3": {
+  "Krull-ramNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "5e05912a316d465f9ebe042b4eb257da261e8b61c2da0df485afe2a977e07bf0",
@@ -2309,7 +2309,7 @@
     "observations": "aa313c1e4cc868d869fff3774cb16a0af7ba5384bedac4b37cb6e99ab625c605",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Krull-v3": {
+  "Krull-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "aa313c1e4cc868d869fff3774cb16a0af7ba5384bedac4b37cb6e99ab625c605",
@@ -2321,7 +2321,7 @@
     "observations": "72364cf936ebbdcac95a53f6cd297bd3756f3e6ec22a9bdcfabf9170c668d273",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "KrullDeterministic-v3": {
+  "KrullDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "72364cf936ebbdcac95a53f6cd297bd3756f3e6ec22a9bdcfabf9170c668d273",
@@ -2333,7 +2333,7 @@
     "observations": "1292e4d52fe0cad2a797c1a0be93469870f9b9efef485b9e40a3b458d5a65fbd",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "KrullNoFrameskip-v3": {
+  "KrullNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "1292e4d52fe0cad2a797c1a0be93469870f9b9efef485b9e40a3b458d5a65fbd",
@@ -2345,7 +2345,7 @@
     "observations": "ee4c4abc1f8b04eb52ae1c7717aac8b96978d8b7d74bc0d6bc71248a54cc5c30",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "KungFuMaster-ram-v3": {
+  "KungFuMaster-ram-v4": {
     "actions": "ae43ac06914f7dab6de7889e1f7b99a91aa10f0204e012bd95e21e929ceda91d",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "ee4c4abc1f8b04eb52ae1c7717aac8b96978d8b7d74bc0d6bc71248a54cc5c30",
@@ -2357,7 +2357,7 @@
     "observations": "b8e85542e913ea80014b1fbf22561f52d4941db2dcb006be6b72d1a658fc371f",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "KungFuMaster-ramDeterministic-v3": {
+  "KungFuMaster-ramDeterministic-v4": {
     "actions": "ae43ac06914f7dab6de7889e1f7b99a91aa10f0204e012bd95e21e929ceda91d",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "c1584450f9352eaa8a543da0beced6fa239d4c59e0bb9203f655a014b9717aba",
@@ -2369,7 +2369,7 @@
     "observations": "816fbc3499c5c0392228beb231263eb06bf3c74debe2fa523b01619e286b378b",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "KungFuMaster-ramNoFrameskip-v3": {
+  "KungFuMaster-ramNoFrameskip-v4": {
     "actions": "ae43ac06914f7dab6de7889e1f7b99a91aa10f0204e012bd95e21e929ceda91d",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "816fbc3499c5c0392228beb231263eb06bf3c74debe2fa523b01619e286b378b",
@@ -2381,7 +2381,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "KungFuMaster-v3": {
+  "KungFuMaster-v4": {
     "actions": "ae43ac06914f7dab6de7889e1f7b99a91aa10f0204e012bd95e21e929ceda91d",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -2393,7 +2393,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "KungFuMasterDeterministic-v3": {
+  "KungFuMasterDeterministic-v4": {
     "actions": "ae43ac06914f7dab6de7889e1f7b99a91aa10f0204e012bd95e21e929ceda91d",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -2405,7 +2405,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "KungFuMasterNoFrameskip-v3": {
+  "KungFuMasterNoFrameskip-v4": {
     "actions": "ae43ac06914f7dab6de7889e1f7b99a91aa10f0204e012bd95e21e929ceda91d",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -2417,7 +2417,7 @@
     "observations": "aff0a6d08f27b19ff9a297559449fb5973299f1e25d382a2802090923471a850",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "MontezumaRevenge-ram-v3": {
+  "MontezumaRevenge-ram-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "54bac54b69049bf8de8cffb8b3c4e57510f6dc318b2d8d95513f3af2a6578e1f",
@@ -2429,7 +2429,7 @@
     "observations": "a62792c9d15845be509c557483b0be9fe7024dfb8f1031692d0c8f4c6aa4d053",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "MontezumaRevenge-ramDeterministic-v3": {
+  "MontezumaRevenge-ramDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "a7c84032d02b5a9bacbeea86010dd6fad7015672de4960008b24ae57dd47eec3",
@@ -2441,7 +2441,7 @@
     "observations": "7934a77785c18a9e3a893c8f248862f7c732abbd76805977a363a466e9986ea1",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "MontezumaRevenge-ramNoFrameskip-v3": {
+  "MontezumaRevenge-ramNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "07636bbdb956a98baa05b6bb36ed0df591ffb4a0f2787c5177d7c90cea9aabb0",
@@ -2453,7 +2453,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "MontezumaRevenge-v3": {
+  "MontezumaRevenge-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -2465,7 +2465,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "MontezumaRevengeDeterministic-v3": {
+  "MontezumaRevengeDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -2477,7 +2477,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "MontezumaRevengeNoFrameskip-v3": {
+  "MontezumaRevengeNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -2501,7 +2501,7 @@
     "observations": "3624b5535998ae8b8cdaf615bcf88ec617a45def211be004282fa6e08066a83f",
     "rewards": "9b3e244462c2706fcd4727350d9779eda7269fcf9840d98a1ecb6d4d0b2859fb"
   },
-  "MsPacman-ram-v3": {
+  "MsPacman-ram-v4": {
     "actions": "680dc83e85ea9c0ec0bed4ba7ae3a87dbf66cc40db1922a0ec9debfca671766f",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "3624b5535998ae8b8cdaf615bcf88ec617a45def211be004282fa6e08066a83f",
@@ -2513,7 +2513,7 @@
     "observations": "6455be4f91789fb011f20c2dfa422d485b1fd194d82bde03165d71243f693276",
     "rewards": "7bea02cb8c558fd4d02adf4cdc0170b0418c043add10c6c6181c512a4a5a526a"
   },
-  "MsPacman-ramDeterministic-v3": {
+  "MsPacman-ramDeterministic-v4": {
     "actions": "680dc83e85ea9c0ec0bed4ba7ae3a87dbf66cc40db1922a0ec9debfca671766f",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "6455be4f91789fb011f20c2dfa422d485b1fd194d82bde03165d71243f693276",
@@ -2525,7 +2525,7 @@
     "observations": "d938cd5f26b6848774b7ba022d39cfb18f740baf103774452c23636b769a8799",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "MsPacman-ramNoFrameskip-v3": {
+  "MsPacman-ramNoFrameskip-v4": {
     "actions": "680dc83e85ea9c0ec0bed4ba7ae3a87dbf66cc40db1922a0ec9debfca671766f",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "d938cd5f26b6848774b7ba022d39cfb18f740baf103774452c23636b769a8799",
@@ -2537,7 +2537,7 @@
     "observations": "10841ba2d1520480b3c0e88673da09ab579cd624fecc7d3ebf063f92c8ecf71c",
     "rewards": "9b3e244462c2706fcd4727350d9779eda7269fcf9840d98a1ecb6d4d0b2859fb"
   },
-  "MsPacman-v3": {
+  "MsPacman-v4": {
     "actions": "680dc83e85ea9c0ec0bed4ba7ae3a87dbf66cc40db1922a0ec9debfca671766f",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "10841ba2d1520480b3c0e88673da09ab579cd624fecc7d3ebf063f92c8ecf71c",
@@ -2549,7 +2549,7 @@
     "observations": "10841ba2d1520480b3c0e88673da09ab579cd624fecc7d3ebf063f92c8ecf71c",
     "rewards": "7bea02cb8c558fd4d02adf4cdc0170b0418c043add10c6c6181c512a4a5a526a"
   },
-  "MsPacmanDeterministic-v3": {
+  "MsPacmanDeterministic-v4": {
     "actions": "680dc83e85ea9c0ec0bed4ba7ae3a87dbf66cc40db1922a0ec9debfca671766f",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "10841ba2d1520480b3c0e88673da09ab579cd624fecc7d3ebf063f92c8ecf71c",
@@ -2561,7 +2561,7 @@
     "observations": "10841ba2d1520480b3c0e88673da09ab579cd624fecc7d3ebf063f92c8ecf71c",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "MsPacmanNoFrameskip-v3": {
+  "MsPacmanNoFrameskip-v4": {
     "actions": "680dc83e85ea9c0ec0bed4ba7ae3a87dbf66cc40db1922a0ec9debfca671766f",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "10841ba2d1520480b3c0e88673da09ab579cd624fecc7d3ebf063f92c8ecf71c",
@@ -2579,7 +2579,7 @@
     "observations": "b5b6335849088f96f426912f573504de4c5f324d46baa5c5c390617a7fa68da1",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "NameThisGame-ram-v3": {
+  "NameThisGame-ram-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "b5b6335849088f96f426912f573504de4c5f324d46baa5c5c390617a7fa68da1",
@@ -2591,7 +2591,7 @@
     "observations": "2f6c829ccafbbe1a914cca54a5868cd3f97cb47beb45b5c48174feb5f8802c24",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "NameThisGame-ramDeterministic-v3": {
+  "NameThisGame-ramDeterministic-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "2f6c829ccafbbe1a914cca54a5868cd3f97cb47beb45b5c48174feb5f8802c24",
@@ -2603,7 +2603,7 @@
     "observations": "a86a00a8b1a1ea062eb276c1b7498d8476fcca8833aefe2ac4541789c6772abf",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "NameThisGame-ramNoFrameskip-v3": {
+  "NameThisGame-ramNoFrameskip-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "a86a00a8b1a1ea062eb276c1b7498d8476fcca8833aefe2ac4541789c6772abf",
@@ -2615,7 +2615,7 @@
     "observations": "5111d143c76defaf223e1bc948a4c339c7b5719f49f98ae257a667006c853d3d",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "NameThisGame-v3": {
+  "NameThisGame-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "5111d143c76defaf223e1bc948a4c339c7b5719f49f98ae257a667006c853d3d",
@@ -2627,7 +2627,7 @@
     "observations": "5111d143c76defaf223e1bc948a4c339c7b5719f49f98ae257a667006c853d3d",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "NameThisGameDeterministic-v3": {
+  "NameThisGameDeterministic-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "5111d143c76defaf223e1bc948a4c339c7b5719f49f98ae257a667006c853d3d",
@@ -2639,7 +2639,7 @@
     "observations": "5111d143c76defaf223e1bc948a4c339c7b5719f49f98ae257a667006c853d3d",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "NameThisGameNoFrameskip-v3": {
+  "NameThisGameNoFrameskip-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "5111d143c76defaf223e1bc948a4c339c7b5719f49f98ae257a667006c853d3d",
@@ -2681,7 +2681,7 @@
     "observations": "6d517be804c98d15ca456670f1e4c3cf800c6621f3aea7f0f0eb310147919a4e",
     "rewards": "4385d91832e2ae017f3f36d99c24679e4cc965d3650eeea12b54e4942d3587ac"
   },
-  "Phoenix-ram-v3": {
+  "Phoenix-ram-v4": {
     "actions": "f72cb9f7a8c584feab60a4f9ae594cbbb98c472df7d917ebf9a20855bec634ae",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "1496e82e6514cf15478829503fdc1243e449eb233daf693ee2376ffd9f879188",
@@ -2693,7 +2693,7 @@
     "observations": "3ab3c4092ce0746fcde491617c8be3893ba16c21590e1c528a117f14e6505390",
     "rewards": "057ed80a52fea497daa5e9b72f6c50231b48a8a20658719d062bfceff90f0924"
   },
-  "Phoenix-ramDeterministic-v3": {
+  "Phoenix-ramDeterministic-v4": {
     "actions": "f72cb9f7a8c584feab60a4f9ae594cbbb98c472df7d917ebf9a20855bec634ae",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "a300c6df1a5104a3c4efb99d8e7238128a1e7e31cfda0d4447f61b0981e8c706",
@@ -2705,7 +2705,7 @@
     "observations": "e84f02eca731d7a2a9fce08e954f4d66c9d00bbe49d3eeb52af8fce6846b42c8",
     "rewards": "42f4485554d0af1389f4d1d4d219e363f988fcc3c8911b7aa6cce6e9ef71e588"
   },
-  "Phoenix-ramNoFrameskip-v3": {
+  "Phoenix-ramNoFrameskip-v4": {
     "actions": "f72cb9f7a8c584feab60a4f9ae594cbbb98c472df7d917ebf9a20855bec634ae",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "4473b3f99a0c33bbe4f5bc9a3b7d02be94bc9b202a7dbe68b6cd52d432e7d5b0",
@@ -2717,7 +2717,7 @@
     "observations": "0680e057e126debb8b8d3106a57293dff8a1003fc396ddaf5740cf5b24e75f2a",
     "rewards": "4385d91832e2ae017f3f36d99c24679e4cc965d3650eeea12b54e4942d3587ac"
   },
-  "Phoenix-v3": {
+  "Phoenix-v4": {
     "actions": "f72cb9f7a8c584feab60a4f9ae594cbbb98c472df7d917ebf9a20855bec634ae",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "0680e057e126debb8b8d3106a57293dff8a1003fc396ddaf5740cf5b24e75f2a",
@@ -2729,7 +2729,7 @@
     "observations": "0680e057e126debb8b8d3106a57293dff8a1003fc396ddaf5740cf5b24e75f2a",
     "rewards": "057ed80a52fea497daa5e9b72f6c50231b48a8a20658719d062bfceff90f0924"
   },
-  "PhoenixDeterministic-v3": {
+  "PhoenixDeterministic-v4": {
     "actions": "f72cb9f7a8c584feab60a4f9ae594cbbb98c472df7d917ebf9a20855bec634ae",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "0680e057e126debb8b8d3106a57293dff8a1003fc396ddaf5740cf5b24e75f2a",
@@ -2741,7 +2741,7 @@
     "observations": "0680e057e126debb8b8d3106a57293dff8a1003fc396ddaf5740cf5b24e75f2a",
     "rewards": "42f4485554d0af1389f4d1d4d219e363f988fcc3c8911b7aa6cce6e9ef71e588"
   },
-  "PhoenixNoFrameskip-v3": {
+  "PhoenixNoFrameskip-v4": {
     "actions": "f72cb9f7a8c584feab60a4f9ae594cbbb98c472df7d917ebf9a20855bec634ae",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "0680e057e126debb8b8d3106a57293dff8a1003fc396ddaf5740cf5b24e75f2a",
@@ -2753,7 +2753,7 @@
     "observations": "79b6876087bddb65d4c7e48ad1b05cdd1437154f9c81710e1b153a6e0c72da5d",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Pitfall-ram-v3": {
+  "Pitfall-ram-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "bdcafcf400167fd821b9027b88bb1749fe31091d494ea0d2327750e8827d0d3f",
@@ -2765,7 +2765,7 @@
     "observations": "5b28cf48edab8d290d789912d265c9acbee00afbb43024b7f79141d0c74b10d9",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Pitfall-ramDeterministic-v3": {
+  "Pitfall-ramDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "c6c5ab9df9c8f848a9c4aaee27ed7298394d47bf79092dba9d038dcf3ad6fee0",
@@ -2777,7 +2777,7 @@
     "observations": "9b07e65d4a0cc12b6d3567ade752961287a78792bf204be0b317cfb9ccf8beea",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Pitfall-ramNoFrameskip-v3": {
+  "Pitfall-ramNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "331fbd32fbf971909c73ea6a2395d65d6797db842cb855fb62a58b2a9cfc25b0",
@@ -2789,7 +2789,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Pitfall-v3": {
+  "Pitfall-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -2801,7 +2801,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "PitfallDeterministic-v3": {
+  "PitfallDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -2813,7 +2813,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "PitfallNoFrameskip-v3": {
+  "PitfallNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -2825,7 +2825,7 @@
     "observations": "b02ffa5443165da029b9ec6c953fa7b7a80e22964d70ab6509dd0ed0f76c148b",
     "rewards": "0be5f310a25bc303c0fa030718593e124eb3de28ec292c702b6e563ff176b6bd"
   },
-  "Pong-ram-v3": {
+  "Pong-ram-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "a7d55350c89bb3a3077b3933f8e628962785ff08f6e9f50f54ae022550125b24",
@@ -2837,7 +2837,7 @@
     "observations": "003785f82b3c6af40729ec10eb85a6ac036fae6dd7322003e335a7ebc4920d49",
     "rewards": "e719c489f04e0a0f3033a2a65f13c86aadf79d4a5209beacfd26025b6552d793"
   },
-  "Pong-ramDeterministic-v3": {
+  "Pong-ramDeterministic-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "9da101c650b6ff5f0163f3735ce9df6d95cba08c8049652013cafbad0e2bf02b",
@@ -2849,7 +2849,7 @@
     "observations": "919a1b1042f88508d9ab88c5577155907f78250a4fa6114aa5558c1054ff61c7",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Pong-ramNoFrameskip-v3": {
+  "Pong-ramNoFrameskip-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "58fd2fae3d605ca9ad44803742d5e5d26f6da74014e43ba0ee39355f266d665f",
@@ -2861,7 +2861,7 @@
     "observations": "4a0ae91640879821611c871b1649c3ae7f708137b50e425b5fe533cdd8064de9",
     "rewards": "0be5f310a25bc303c0fa030718593e124eb3de28ec292c702b6e563ff176b6bd"
   },
-  "Pong-v3": {
+  "Pong-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "4a0ae91640879821611c871b1649c3ae7f708137b50e425b5fe533cdd8064de9",
@@ -2873,7 +2873,7 @@
     "observations": "4a0ae91640879821611c871b1649c3ae7f708137b50e425b5fe533cdd8064de9",
     "rewards": "e719c489f04e0a0f3033a2a65f13c86aadf79d4a5209beacfd26025b6552d793"
   },
-  "PongDeterministic-v3": {
+  "PongDeterministic-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "4a0ae91640879821611c871b1649c3ae7f708137b50e425b5fe533cdd8064de9",
@@ -2885,7 +2885,7 @@
     "observations": "4a834d5fd0bfff0395100c2c03efcbf2e29e04a5825216430be1d3b69e9c8038",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "PongNoFrameskip-v3": {
+  "PongNoFrameskip-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "4a834d5fd0bfff0395100c2c03efcbf2e29e04a5825216430be1d3b69e9c8038",
@@ -2897,7 +2897,7 @@
     "observations": "f503d9f15020a7aefa685289dfa4192f6f0eb520414520f1d8a1efcc6062ee25",
     "rewards": "6e7eddbc98a7e3fa49a6019c1500c2c7af61497fdbf034c8e2495f8299c3ee31"
   },
-  "Pooyan-ram-v3": {
+  "Pooyan-ram-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "3d66f26edc8933058979855e7385814b13d1021d3b8f6db51ff6910274a8015b",
@@ -2909,7 +2909,7 @@
     "observations": "154cf321c5a49b50a6304a1840e089fa1818a33c433fb225e01484e8040c76a3",
     "rewards": "982e0adff65defd4dccda90659b0b03bd8488cd3af3a800d2fbed0150467f0a6"
   },
-  "Pooyan-ramDeterministic-v3": {
+  "Pooyan-ramDeterministic-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "9465505ac6f28607dda66cc03c6d3050cafd70b0d680636dd6a848564c982137",
@@ -2921,7 +2921,7 @@
     "observations": "fc20190021106c65925fb6cad8e53047183d9d009c9e6acdb99a1ffdc4f13406",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Pooyan-ramNoFrameskip-v3": {
+  "Pooyan-ramNoFrameskip-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "fc20190021106c65925fb6cad8e53047183d9d009c9e6acdb99a1ffdc4f13406",
@@ -2933,7 +2933,7 @@
     "observations": "9c4ea9c8b9063dd3a14a93a6d4e0f24226249feeffcc4579bb2a97b90b3bbdd2",
     "rewards": "6e7eddbc98a7e3fa49a6019c1500c2c7af61497fdbf034c8e2495f8299c3ee31"
   },
-  "Pooyan-v3": {
+  "Pooyan-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "9c4ea9c8b9063dd3a14a93a6d4e0f24226249feeffcc4579bb2a97b90b3bbdd2",
@@ -2945,7 +2945,7 @@
     "observations": "48f4481190ff1ee523e09966cb6710346d32142c47f06d7f230a198c0262ec4d",
     "rewards": "982e0adff65defd4dccda90659b0b03bd8488cd3af3a800d2fbed0150467f0a6"
   },
-  "PooyanDeterministic-v3": {
+  "PooyanDeterministic-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "48f4481190ff1ee523e09966cb6710346d32142c47f06d7f230a198c0262ec4d",
@@ -2957,7 +2957,7 @@
     "observations": "b8a6b56dd00786c942c710e6d604e0df27f82e17dac22e789bed27fb8e9443dc",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "PooyanNoFrameskip-v3": {
+  "PooyanNoFrameskip-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "b8a6b56dd00786c942c710e6d604e0df27f82e17dac22e789bed27fb8e9443dc",
@@ -2981,7 +2981,7 @@
     "observations": "673397a3bfea1c0d04408caaac6c4a911ac10e64aa9488c77d30d7a5ad819829",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "PrivateEye-ram-v3": {
+  "PrivateEye-ram-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "a4696cb0a77e75f3e49ffeae64999fbf713ffcb44a2cdeb83612681efef6514f",
@@ -2993,7 +2993,7 @@
     "observations": "f9e072cbe0f7686d9559bb21cb45b4cd4c7791d73101c26ced37c11900bef680",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "PrivateEye-ramDeterministic-v3": {
+  "PrivateEye-ramDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "71a0d2462d2e6b4ada96eda7afea71fee45568a6a1e3094a5a44bf879e4300a8",
@@ -3005,7 +3005,7 @@
     "observations": "2ea827df5ce3cc030fb7a3f0fe9674e1150a99bdc2d0edf37908372b314afcbe",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "PrivateEye-ramNoFrameskip-v3": {
+  "PrivateEye-ramNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "055d1bc8fde531cb4dbe05a12352c8df54d084f24a66a02cd6e6301ba247211b",
@@ -3017,7 +3017,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "PrivateEye-v3": {
+  "PrivateEye-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -3029,7 +3029,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "PrivateEyeDeterministic-v3": {
+  "PrivateEyeDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -3041,7 +3041,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "PrivateEyeNoFrameskip-v3": {
+  "PrivateEyeNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -3053,7 +3053,7 @@
     "observations": "a8bcda751ff0be6066515a11ec0700f60b0b53d5b6916fad5711577398e0aaa5",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Qbert-ram-v3": {
+  "Qbert-ram-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "a8bcda751ff0be6066515a11ec0700f60b0b53d5b6916fad5711577398e0aaa5",
@@ -3065,7 +3065,7 @@
     "observations": "4a4f641579d031c9b840ba2bac4c9a73ada905a609638d91270424124eb0b2e8",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Qbert-ramDeterministic-v3": {
+  "Qbert-ramDeterministic-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "e0b9de3a1714361582787aaf22b6d57700caa11dfa037d41610361972613dedb",
@@ -3077,7 +3077,7 @@
     "observations": "16c18e837a4f97bd4ce5fcd7c18b142ea9f80e9249ba3a46a871cc45d28d0dbb",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Qbert-ramNoFrameskip-v3": {
+  "Qbert-ramNoFrameskip-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "16c18e837a4f97bd4ce5fcd7c18b142ea9f80e9249ba3a46a871cc45d28d0dbb",
@@ -3089,7 +3089,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Qbert-v3": {
+  "Qbert-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -3101,7 +3101,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "QbertDeterministic-v3": {
+  "QbertDeterministic-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -3113,7 +3113,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "QbertNoFrameskip-v3": {
+  "QbertNoFrameskip-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -3149,7 +3149,7 @@
     "observations": "05addc90a2a2b60d9d71c5211f3d60330692349a80ceeb20801aa529c4dc634b",
     "rewards": "5f72f29daf423adad0018a8f5c8859bde026c80d58e7c879fbf0465a870b8cb6"
   },
-  "Riverraid-ram-v3": {
+  "Riverraid-ram-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "e25fce00b2f43c724b1004d7729380fa75c4e53d7cb1a409e718a2510b392a3e",
@@ -3161,7 +3161,7 @@
     "observations": "1302106374f809e5b5cbc67588d2eb7a181c15d73bec3f627f59d107fe2497d3",
     "rewards": "746a946619a383e1901bfdbdd76cb2c5a14de1fd94d5818a89f279072f537011"
   },
-  "Riverraid-ramDeterministic-v3": {
+  "Riverraid-ramDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "89899a8e8c53de4ea7fbc7a58159a6a6a8fdb5f07ef08b8fc896098420123011",
@@ -3173,7 +3173,7 @@
     "observations": "848c9812620f4e0d012fe79b76ac34aa010d5387d13675221194724407b9d7cd",
     "rewards": "f7d2a3cdcbcf7882910389eb4cebfbe26fbf135c6591950c55d0aaed778b3718"
   },
-  "Riverraid-ramNoFrameskip-v3": {
+  "Riverraid-ramNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "6bacadbb2c71eb44b7e37650b5d1cdd24ef1f9fd49e8afc1317df7459f71a579",
@@ -3185,7 +3185,7 @@
     "observations": "f6a50d170699a2ce2462b4415e5676b130c8e5cdb24a62800ff8714edfb3725e",
     "rewards": "5f72f29daf423adad0018a8f5c8859bde026c80d58e7c879fbf0465a870b8cb6"
   },
-  "Riverraid-v3": {
+  "Riverraid-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "f6a50d170699a2ce2462b4415e5676b130c8e5cdb24a62800ff8714edfb3725e",
@@ -3197,7 +3197,7 @@
     "observations": "53ffde2c1904b560e4126ef29f51d301f65e9d89de9b0448bf6ba0cdffc9bf7e",
     "rewards": "746a946619a383e1901bfdbdd76cb2c5a14de1fd94d5818a89f279072f537011"
   },
-  "RiverraidDeterministic-v3": {
+  "RiverraidDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "53ffde2c1904b560e4126ef29f51d301f65e9d89de9b0448bf6ba0cdffc9bf7e",
@@ -3209,7 +3209,7 @@
     "observations": "6806f6cbb36d15f469d701c4632bb9a589dbcb7ad5904af58edb6bd24aea6a43",
     "rewards": "f7d2a3cdcbcf7882910389eb4cebfbe26fbf135c6591950c55d0aaed778b3718"
   },
-  "RiverraidNoFrameskip-v3": {
+  "RiverraidNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "6806f6cbb36d15f469d701c4632bb9a589dbcb7ad5904af58edb6bd24aea6a43",
@@ -3221,7 +3221,7 @@
     "observations": "461fee4c6ef4f59a7d60bea30fdafef3d991de6ab4f16a6a88ac5fa3b4380704",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "RoadRunner-ram-v3": {
+  "RoadRunner-ram-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "5247a0cd43528eb421e89e2de2b7b351400d99a4df82309a6d8981ac1260d5c3",
@@ -3233,7 +3233,7 @@
     "observations": "5655619ca83c9a131ede19f4938c63867da2943c6f3663f99eda05db8c64e51a",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "RoadRunner-ramDeterministic-v3": {
+  "RoadRunner-ramDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "7e8d7fdd03e461df91b04758ca3ce9ddbb01ef8fb857de69a812ef2ac3d51862",
@@ -3245,7 +3245,7 @@
     "observations": "6a906213cb06335b516f9e8f68b00b23ba706d0561f2147d6128c74dae2b215d",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "RoadRunner-ramNoFrameskip-v3": {
+  "RoadRunner-ramNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "5614cbc8a2b837cea183fbda95706bc247dd245d4939906643896f726cf9310a",
@@ -3257,7 +3257,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "RoadRunner-v3": {
+  "RoadRunner-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -3269,7 +3269,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "RoadRunnerDeterministic-v3": {
+  "RoadRunnerDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -3281,7 +3281,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "RoadRunnerNoFrameskip-v3": {
+  "RoadRunnerNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -3293,7 +3293,7 @@
     "observations": "fa789b70fe82f164db1892b3a33c89b0a4e8a6baaf99b79e1415274d795ab2b9",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Robotank-ram-v3": {
+  "Robotank-ram-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "a126392d3b2107feda9ca7a651781fd61493527dd7af16971d18e58d0b0c1bb9",
@@ -3305,7 +3305,7 @@
     "observations": "be9ea03ede6484b07e434cdc23bf58d232c4cb5546a6ee6ae9deb26026266106",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Robotank-ramDeterministic-v3": {
+  "Robotank-ramDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "a1d93c9883a88fb082e603f77cd20e30d36302c0a970e08c9d51c691def9a2b0",
@@ -3317,7 +3317,7 @@
     "observations": "b0da08e37512ccf2ffe43c205204a7341810a39b7989b8295f85a665b6c33553",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Robotank-ramNoFrameskip-v3": {
+  "Robotank-ramNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "9f7a52985b86d62e0664d197c7e3fbcae0c027a6f57f1496c2cacf710fc8dc41",
@@ -3329,7 +3329,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Robotank-v3": {
+  "Robotank-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -3341,7 +3341,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "RobotankDeterministic-v3": {
+  "RobotankDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -3353,7 +3353,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "RobotankNoFrameskip-v3": {
+  "RobotankNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -3371,7 +3371,7 @@
     "observations": "14c92d45a119a22a635d8adc4c9d15f1fcfc3a1581c6fa993ef2545fe49305a6",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Seaquest-ram-v3": {
+  "Seaquest-ram-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "9038139d24a77fb86789aaf2333e0c25e313a4164b14e255d614ceae585d293d",
@@ -3383,7 +3383,7 @@
     "observations": "07d4db3caf9f5061ecc4f2293ef5680eca48e588929bee72deec0b839ad51fd8",
     "rewards": "4b7125648606f6a1f35de5916d8a3369dfd6d4e4ce41883f915a6931c70ad09c"
   },
-  "Seaquest-ramDeterministic-v3": {
+  "Seaquest-ramDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "dc6585d57f3502ea55870f82be6180d62ad7aac36bfac2579e1e9afe521f44b5",
@@ -3395,7 +3395,7 @@
     "observations": "38a31ed9d778e177837372f6965c286796024d10c83839a0e8403a13ce6a1b45",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Seaquest-ramNoFrameskip-v3": {
+  "Seaquest-ramNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "3af97033eb71e6d114b50670a8d0f681c80c94669e5f8968ad40e7382509d105",
@@ -3407,7 +3407,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Seaquest-v3": {
+  "Seaquest-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -3419,7 +3419,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "4b7125648606f6a1f35de5916d8a3369dfd6d4e4ce41883f915a6931c70ad09c"
   },
-  "SeaquestDeterministic-v3": {
+  "SeaquestDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -3431,7 +3431,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "SeaquestNoFrameskip-v3": {
+  "SeaquestNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -3455,7 +3455,7 @@
     "observations": "1c68c3338878b5d7c3e31816ca8ffa27aeee67636b7c3a23915240d19d8c55b8",
     "rewards": "83da542fdf7e0eb1829764bb7cfe8e499fcae2951b3a5022c8622e4a50880fac"
   },
-  "Skiing-ram-v3": {
+  "Skiing-ram-v4": {
     "actions": "5138748c3c039a57ee365473ef13e5b99329e75a4f71459cd1a0d7919fd6e97b",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "e7cb8d20e5cebe5ce21bc83e90f455c872ca8bd1a4787aa4f7812171dac05d89",
@@ -3467,7 +3467,7 @@
     "observations": "b4e7c5a42eb8c45839e8b1f2590ffd3a097b78868de6753da780e9693e7fab5a",
     "rewards": "56662f81ec8ca11dc136ab75158deb085ee4b683d03ebc3ec56aea9763ec85f9"
   },
-  "Skiing-ramDeterministic-v3": {
+  "Skiing-ramDeterministic-v4": {
     "actions": "5138748c3c039a57ee365473ef13e5b99329e75a4f71459cd1a0d7919fd6e97b",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "9a330ead57c50c42edaf5c1eb7a49a996e01207fbd7572711bb993a106b7f913",
@@ -3479,7 +3479,7 @@
     "observations": "a562df7c780e87fc251bfd7fa8fcfb944791ae9ce56f64b3fba0c8de7bbddf7c",
     "rewards": "d1a2c71ad5a6c1cdaff572ea9121f9154e5efa1dc9339e3d441de64dc1ac7890"
   },
-  "Skiing-ramNoFrameskip-v3": {
+  "Skiing-ramNoFrameskip-v4": {
     "actions": "5138748c3c039a57ee365473ef13e5b99329e75a4f71459cd1a0d7919fd6e97b",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "212da5d4f910803f8f937e0637ed48ad1bf50950ef3b6b8441ab39446c9766ff",
@@ -3491,7 +3491,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "83da542fdf7e0eb1829764bb7cfe8e499fcae2951b3a5022c8622e4a50880fac"
   },
-  "Skiing-v3": {
+  "Skiing-v4": {
     "actions": "5138748c3c039a57ee365473ef13e5b99329e75a4f71459cd1a0d7919fd6e97b",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -3503,7 +3503,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "56662f81ec8ca11dc136ab75158deb085ee4b683d03ebc3ec56aea9763ec85f9"
   },
-  "SkiingDeterministic-v3": {
+  "SkiingDeterministic-v4": {
     "actions": "5138748c3c039a57ee365473ef13e5b99329e75a4f71459cd1a0d7919fd6e97b",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -3515,7 +3515,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "d1a2c71ad5a6c1cdaff572ea9121f9154e5efa1dc9339e3d441de64dc1ac7890"
   },
-  "SkiingNoFrameskip-v3": {
+  "SkiingNoFrameskip-v4": {
     "actions": "5138748c3c039a57ee365473ef13e5b99329e75a4f71459cd1a0d7919fd6e97b",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -3527,7 +3527,7 @@
     "observations": "dbffdd6bf82f7996b051b0079d485c529731978582ff6505bb6c49eff5c15ed7",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Solaris-ram-v3": {
+  "Solaris-ram-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "a0f566d393fd6af3da70ca8c6ee72fd6fa6bb6245d69ed796e6eb60330d1982c",
@@ -3539,7 +3539,7 @@
     "observations": "69ec50e0e271cd2e352891a15963b20db9f80db72d6ade8a927e242d28390461",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Solaris-ramDeterministic-v3": {
+  "Solaris-ramDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "78dd52c2007c12755f9761978f5f46f4b0dee109f57d9fe8f6d8a0b7412e9017",
@@ -3551,7 +3551,7 @@
     "observations": "d1cf36839d9d57c2fcbafa9991646fcc2fa44205f2af986ee3336b76e423c487",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Solaris-ramNoFrameskip-v3": {
+  "Solaris-ramNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "f21d7e7c85c6dd08fdeece4e1bec6a7c200e4fcb0696cd18fa32ebe01bdaf6b2",
@@ -3563,7 +3563,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Solaris-v3": {
+  "Solaris-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -3575,7 +3575,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "SolarisDeterministic-v3": {
+  "SolarisDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -3587,7 +3587,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "SolarisNoFrameskip-v3": {
+  "SolarisNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -3599,7 +3599,7 @@
     "observations": "c4198a4d7aeec394ff04d11c160b40c6a969650b6457d6b25721bd233a562871",
     "rewards": "537e7dd984f475e6014007d87091e94466bbaf09fcb2c609a0e1e9b6939c2ca8"
   },
-  "SpaceInvaders-ram-v3": {
+  "SpaceInvaders-ram-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "da04f2510b32b63dd7ebeaf7d0e7e7ef15d755285f6772f09ccbb85b7131f327",
@@ -3611,7 +3611,7 @@
     "observations": "a5d60799ec76ad8f9e0b2528ec209ed3177bcd8a7848b7fe5a87b40cd0f01bd1",
     "rewards": "b05c1ddf7aca53614543edbda9095fdfd98a017fa48d74dd5848b7a3cc184ec6"
   },
-  "SpaceInvaders-ramDeterministic-v3": {
+  "SpaceInvaders-ramDeterministic-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "5f794711c857aedd99d5803202e82ce7653f245e22ea935b60af227a35a79b3b",
@@ -3623,7 +3623,7 @@
     "observations": "f30c57e8cc5a455e20748068fbfd92fa1ed091c3d332c1c95dc1338ae93df1a4",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "SpaceInvaders-ramNoFrameskip-v3": {
+  "SpaceInvaders-ramNoFrameskip-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "a9da1911021b7856b26de156b12465a319a0856e38580a9794e49a410f11fa28",
@@ -3635,7 +3635,7 @@
     "observations": "de5a2ccf1c3e790b80db358183291e18bd9ab834c06d0a4d3f8fad7340e89ed5",
     "rewards": "537e7dd984f475e6014007d87091e94466bbaf09fcb2c609a0e1e9b6939c2ca8"
   },
-  "SpaceInvaders-v3": {
+  "SpaceInvaders-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "de5a2ccf1c3e790b80db358183291e18bd9ab834c06d0a4d3f8fad7340e89ed5",
@@ -3647,7 +3647,7 @@
     "observations": "de5a2ccf1c3e790b80db358183291e18bd9ab834c06d0a4d3f8fad7340e89ed5",
     "rewards": "b05c1ddf7aca53614543edbda9095fdfd98a017fa48d74dd5848b7a3cc184ec6"
   },
-  "SpaceInvadersDeterministic-v3": {
+  "SpaceInvadersDeterministic-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "de5a2ccf1c3e790b80db358183291e18bd9ab834c06d0a4d3f8fad7340e89ed5",
@@ -3659,7 +3659,7 @@
     "observations": "de5a2ccf1c3e790b80db358183291e18bd9ab834c06d0a4d3f8fad7340e89ed5",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "SpaceInvadersNoFrameskip-v3": {
+  "SpaceInvadersNoFrameskip-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "de5a2ccf1c3e790b80db358183291e18bd9ab834c06d0a4d3f8fad7340e89ed5",
@@ -3671,7 +3671,7 @@
     "observations": "ec3a35f4a520c67e7bf519ff320e0c602c50d8ab945962d24f25a1661b360ea3",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "StarGunner-ram-v3": {
+  "StarGunner-ram-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "7d36d67ea7308c0e0e1d7862d72a261144737aec830c26ce4df220e246ddc0e6",
@@ -3683,7 +3683,7 @@
     "observations": "c9de3460b4ed0434e2315cd4bc96812944892fdb06cb4f2c1c8fce8a989c09bd",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "StarGunner-ramDeterministic-v3": {
+  "StarGunner-ramDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "2f13dce1df52db99b8b118b91e9b4a331e29a31c0ffdc4c80057ba1cf817aabc",
@@ -3695,7 +3695,7 @@
     "observations": "d5e8424f3e4df54e49cca69bba870be303257ee517dc5c87cf1814d2d4a552c8",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "StarGunner-ramNoFrameskip-v3": {
+  "StarGunner-ramNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "d5e8424f3e4df54e49cca69bba870be303257ee517dc5c87cf1814d2d4a552c8",
@@ -3707,7 +3707,7 @@
     "observations": "dc6a89cebe2307516a293b41439499bc899adeca63abddd0ebd36b042355bafb",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "StarGunner-v3": {
+  "StarGunner-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "dc6a89cebe2307516a293b41439499bc899adeca63abddd0ebd36b042355bafb",
@@ -3719,7 +3719,7 @@
     "observations": "dc6a89cebe2307516a293b41439499bc899adeca63abddd0ebd36b042355bafb",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "StarGunnerDeterministic-v3": {
+  "StarGunnerDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "dc6a89cebe2307516a293b41439499bc899adeca63abddd0ebd36b042355bafb",
@@ -3731,7 +3731,7 @@
     "observations": "dc6a89cebe2307516a293b41439499bc899adeca63abddd0ebd36b042355bafb",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "StarGunnerNoFrameskip-v3": {
+  "StarGunnerNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "dc6a89cebe2307516a293b41439499bc899adeca63abddd0ebd36b042355bafb",
@@ -3749,7 +3749,7 @@
     "observations": "8ada333e8b52ce176bd5a10e7575f1cf4dee37719c70f18910a731b1b5249200",
     "rewards": "59c4cb21bf749812c1e8aec0106fa7d2b2c98c76c16ff4507904f14b29c00d09"
   },
-  "Tennis-ram-v3": {
+  "Tennis-ram-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "f143b0d1129bb44e5ceca90ddb2d60e167de756dad61b663ed65d4f389e0e925",
@@ -3761,7 +3761,7 @@
     "observations": "e7af17e04d1881ce77635e94ad85fe6243302f09243fdd27a87ba7e568702aac",
     "rewards": "ed52a5f814fb082a3010fd7ca5e5d4798c9aaaf11011156c2ad26ad180f54717"
   },
-  "Tennis-ramDeterministic-v3": {
+  "Tennis-ramDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "60a2392d57c46d8ba8095a0f417c6f7e00668ba9806adb8f0f9f7fa6a382a9b7",
@@ -3773,7 +3773,7 @@
     "observations": "b90465e1bc029ba63084e5b6fd918ada99cf23cd6b3cdc8a281da5bb8fb3bf04",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Tennis-ramNoFrameskip-v3": {
+  "Tennis-ramNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "65e90120376a40fe51e52938631ce5e9b27ab1a2dd93cce92404380fd28b25a2",
@@ -3785,7 +3785,7 @@
     "observations": "2c584c6be5aea0fe9db4a2fdfda524f536d982bb55437f75dbae3d61430238d0",
     "rewards": "59c4cb21bf749812c1e8aec0106fa7d2b2c98c76c16ff4507904f14b29c00d09"
   },
-  "Tennis-v3": {
+  "Tennis-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "2c584c6be5aea0fe9db4a2fdfda524f536d982bb55437f75dbae3d61430238d0",
@@ -3797,7 +3797,7 @@
     "observations": "2c584c6be5aea0fe9db4a2fdfda524f536d982bb55437f75dbae3d61430238d0",
     "rewards": "ed52a5f814fb082a3010fd7ca5e5d4798c9aaaf11011156c2ad26ad180f54717"
   },
-  "TennisDeterministic-v3": {
+  "TennisDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "2c584c6be5aea0fe9db4a2fdfda524f536d982bb55437f75dbae3d61430238d0",
@@ -3809,7 +3809,7 @@
     "observations": "2c584c6be5aea0fe9db4a2fdfda524f536d982bb55437f75dbae3d61430238d0",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "TennisNoFrameskip-v3": {
+  "TennisNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "2c584c6be5aea0fe9db4a2fdfda524f536d982bb55437f75dbae3d61430238d0",
@@ -3821,7 +3821,7 @@
     "observations": "95b9064774c6a60c14932911f76f974f1f943887eb65062e4bc964122274cf31",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "TimePilot-ram-v3": {
+  "TimePilot-ram-v4": {
     "actions": "b199b81b77e4e2a8aad9a5663315bd9f7a65ba9ad191c7f8645848e7291df62e",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "95b9064774c6a60c14932911f76f974f1f943887eb65062e4bc964122274cf31",
@@ -3833,7 +3833,7 @@
     "observations": "52b574c8c66020b49509dd52e16095aab78dcd8028421f10fd5380d7014c3295",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "TimePilot-ramDeterministic-v3": {
+  "TimePilot-ramDeterministic-v4": {
     "actions": "b199b81b77e4e2a8aad9a5663315bd9f7a65ba9ad191c7f8645848e7291df62e",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "52b574c8c66020b49509dd52e16095aab78dcd8028421f10fd5380d7014c3295",
@@ -3845,7 +3845,7 @@
     "observations": "0cd9ab26edb8e67eb4235217c6f6ca576d0dca20b12b129e964a2ea92a21d5e8",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "TimePilot-ramNoFrameskip-v3": {
+  "TimePilot-ramNoFrameskip-v4": {
     "actions": "b199b81b77e4e2a8aad9a5663315bd9f7a65ba9ad191c7f8645848e7291df62e",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "0cd9ab26edb8e67eb4235217c6f6ca576d0dca20b12b129e964a2ea92a21d5e8",
@@ -3857,7 +3857,7 @@
     "observations": "04d522742a6a56e859848194ebb7670056dde78f04bd97911799b39e5be04bde",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "TimePilot-v3": {
+  "TimePilot-v4": {
     "actions": "b199b81b77e4e2a8aad9a5663315bd9f7a65ba9ad191c7f8645848e7291df62e",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "04d522742a6a56e859848194ebb7670056dde78f04bd97911799b39e5be04bde",
@@ -3869,7 +3869,7 @@
     "observations": "6bbead75c397d9d2ec407dbb36caa99a10561746129ffd25465b131456e0575a",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "TimePilotDeterministic-v3": {
+  "TimePilotDeterministic-v4": {
     "actions": "b199b81b77e4e2a8aad9a5663315bd9f7a65ba9ad191c7f8645848e7291df62e",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "6bbead75c397d9d2ec407dbb36caa99a10561746129ffd25465b131456e0575a",
@@ -3881,7 +3881,7 @@
     "observations": "96cdcfb80ba7a249679fd0307ac3fdc9223262d970f85490a1e6ca9e674c10e0",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "TimePilotNoFrameskip-v3": {
+  "TimePilotNoFrameskip-v4": {
     "actions": "b199b81b77e4e2a8aad9a5663315bd9f7a65ba9ad191c7f8645848e7291df62e",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "96cdcfb80ba7a249679fd0307ac3fdc9223262d970f85490a1e6ca9e674c10e0",
@@ -3893,7 +3893,7 @@
     "observations": "9221579872313e42b7099257b38896e3b7ea78937c71b15b60e3d4c2d2d95968",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Tutankham-ram-v3": {
+  "Tutankham-ram-v4": {
     "actions": "f72cb9f7a8c584feab60a4f9ae594cbbb98c472df7d917ebf9a20855bec634ae",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "f5c757343a5f423a1630030ec6d888d5215e9c6625bf5c69597035b0f2eb867d",
@@ -3905,7 +3905,7 @@
     "observations": "e73d33ec89e8ca0523ed9a727df2f7c6b7eb399ceaabc35cc40ed46b6d40017f",
     "rewards": "a93d54e993a09ec3497890a90513b59f4dc30bf00d4cb0e9d461756344fb4ef2"
   },
-  "Tutankham-ramDeterministic-v3": {
+  "Tutankham-ramDeterministic-v4": {
     "actions": "f72cb9f7a8c584feab60a4f9ae594cbbb98c472df7d917ebf9a20855bec634ae",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "b0f39d8754967a9a151a37a040b181fafcbcd56e022b82db4b9a692918efe604",
@@ -3917,7 +3917,7 @@
     "observations": "680854c46b05b6ad151aa897a78ed728db6fd8b2d9d92c351cd14e7c138c04e5",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Tutankham-ramNoFrameskip-v3": {
+  "Tutankham-ramNoFrameskip-v4": {
     "actions": "f72cb9f7a8c584feab60a4f9ae594cbbb98c472df7d917ebf9a20855bec634ae",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "346ec7e445deb71b4c106cc65a909811b21de6a117671d9a2c2a58692a26b09b",
@@ -3929,7 +3929,7 @@
     "observations": "aa843ae315a43e08358abc8ee2625c2a16a7d5813816fefbef17e673a5a1f5c7",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Tutankham-v3": {
+  "Tutankham-v4": {
     "actions": "f72cb9f7a8c584feab60a4f9ae594cbbb98c472df7d917ebf9a20855bec634ae",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "aa843ae315a43e08358abc8ee2625c2a16a7d5813816fefbef17e673a5a1f5c7",
@@ -3941,7 +3941,7 @@
     "observations": "aa843ae315a43e08358abc8ee2625c2a16a7d5813816fefbef17e673a5a1f5c7",
     "rewards": "a93d54e993a09ec3497890a90513b59f4dc30bf00d4cb0e9d461756344fb4ef2"
   },
-  "TutankhamDeterministic-v3": {
+  "TutankhamDeterministic-v4": {
     "actions": "f72cb9f7a8c584feab60a4f9ae594cbbb98c472df7d917ebf9a20855bec634ae",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "aa843ae315a43e08358abc8ee2625c2a16a7d5813816fefbef17e673a5a1f5c7",
@@ -3953,7 +3953,7 @@
     "observations": "aa843ae315a43e08358abc8ee2625c2a16a7d5813816fefbef17e673a5a1f5c7",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "TutankhamNoFrameskip-v3": {
+  "TutankhamNoFrameskip-v4": {
     "actions": "f72cb9f7a8c584feab60a4f9ae594cbbb98c472df7d917ebf9a20855bec634ae",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "aa843ae315a43e08358abc8ee2625c2a16a7d5813816fefbef17e673a5a1f5c7",
@@ -3977,7 +3977,7 @@
     "observations": "4296611c75769070866c9cc35f06bb661ddc16cd87c7bbab47730dcda70890e9",
     "rewards": "535b31f3f6a04ef863b22634435328dda9e5b49c810c2ebda398a55e801c256e"
   },
-  "UpNDown-ram-v3": {
+  "UpNDown-ram-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "f85ebbec68c53b9428d269763f88321eeb8a9c0ae36a1fdf09be99431b178080",
@@ -3989,7 +3989,7 @@
     "observations": "8ac0a2208796cac4fdc93e716a0e1ecbae3e94af3c0f94501815b977b5cbd57e",
     "rewards": "51b8beb1f4ddd5c05d3abad991f0e3f9841e8fb404956ce30ef0a55da002e9e9"
   },
-  "UpNDown-ramDeterministic-v3": {
+  "UpNDown-ramDeterministic-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "df5a1845a6f5ac0bef87242412d4169d58f07714c1ff42c15632910229c7833a",
@@ -4001,7 +4001,7 @@
     "observations": "e3d9c2690900c6b079816e4b0faceff7103f610b9a55ddd0e76e09255f9cbd3d",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "UpNDown-ramNoFrameskip-v3": {
+  "UpNDown-ramNoFrameskip-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "debe1dcd844a36e88d88ef97b5a0c8eaa01010e8082ccb89510d670c24c76168",
@@ -4013,7 +4013,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "535b31f3f6a04ef863b22634435328dda9e5b49c810c2ebda398a55e801c256e"
   },
-  "UpNDown-v3": {
+  "UpNDown-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -4025,7 +4025,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "51b8beb1f4ddd5c05d3abad991f0e3f9841e8fb404956ce30ef0a55da002e9e9"
   },
-  "UpNDownDeterministic-v3": {
+  "UpNDownDeterministic-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -4037,7 +4037,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "UpNDownNoFrameskip-v3": {
+  "UpNDownNoFrameskip-v4": {
     "actions": "7364c36f0f18ebecf3d6086b3e09a8944af50d3f40f25c2efb338bc42cc7255a",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -4049,7 +4049,7 @@
     "observations": "e91770ad63927d7815da4d8ef4bb97d0e8029e36aac026df5f62a66c68c02004",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Venture-ram-v3": {
+  "Venture-ram-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "2818ccd46779ed0dc94485aa3b1c6ec3c265edd919ae7c441d7aadbb8bb7a2a5",
@@ -4061,7 +4061,7 @@
     "observations": "ce122765f56b3f7909f628656bf9ed8b4d033799aabc642b57260752733cd4ab",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Venture-ramDeterministic-v3": {
+  "Venture-ramDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "dbee5071b192fb25e54815b74e03b5d4e53544bcdc289cb51e9998c13089d8d7",
@@ -4073,7 +4073,7 @@
     "observations": "29a85453075378cc16a1b8b7feaab359a678dfccf85d053ab0a9dc08850c69fc",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Venture-ramNoFrameskip-v3": {
+  "Venture-ramNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "6f7a689948749b10095c9926374802584e1740e3e0ae460915c03f2ec4858e84",
@@ -4085,7 +4085,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Venture-v3": {
+  "Venture-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -4097,7 +4097,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "VentureDeterministic-v3": {
+  "VentureDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -4109,7 +4109,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "VentureNoFrameskip-v3": {
+  "VentureNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -4121,7 +4121,7 @@
     "observations": "f3a7c8ed344d53bc26fdf8b3833cd55b5f9398c08663e41da80365c2622d0af1",
     "rewards": "86a5ee144f3fa5812bf836306529afd206122cc790406510d32a7a6fc240cb29"
   },
-  "VideoPinball-ram-v3": {
+  "VideoPinball-ram-v4": {
     "actions": "680dc83e85ea9c0ec0bed4ba7ae3a87dbf66cc40db1922a0ec9debfca671766f",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "1374718b30614e592c4225e9d0c9b165545759bd9d9bbcc5371e3836a130a6a5",
@@ -4133,7 +4133,7 @@
     "observations": "71f905bf2771ec372a912c117dac59a682c760102dc9bf35d0c9d2a0d6ce87c8",
     "rewards": "d2f61482ccc9112d2606b8d361fb533798f79ea0a93370c7171819be144e5aca"
   },
-  "VideoPinball-ramDeterministic-v3": {
+  "VideoPinball-ramDeterministic-v4": {
     "actions": "680dc83e85ea9c0ec0bed4ba7ae3a87dbf66cc40db1922a0ec9debfca671766f",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "f3fe66bde105218c9ec8c777cd521597a6a1035e98e5f02a3feab96d052847ea",
@@ -4145,7 +4145,7 @@
     "observations": "6710ee57b79875e9f3979fca21c382ef8da34339b5e3d39a4d929925cbf35cdd",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "VideoPinball-ramNoFrameskip-v3": {
+  "VideoPinball-ramNoFrameskip-v4": {
     "actions": "680dc83e85ea9c0ec0bed4ba7ae3a87dbf66cc40db1922a0ec9debfca671766f",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "e99185a45bbc50c0fc2ad8b70ddb499ed25229bc6909b59b15adb1b9cbebfb79",
@@ -4157,7 +4157,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "86a5ee144f3fa5812bf836306529afd206122cc790406510d32a7a6fc240cb29"
   },
-  "VideoPinball-v3": {
+  "VideoPinball-v4": {
     "actions": "680dc83e85ea9c0ec0bed4ba7ae3a87dbf66cc40db1922a0ec9debfca671766f",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -4169,7 +4169,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "d2f61482ccc9112d2606b8d361fb533798f79ea0a93370c7171819be144e5aca"
   },
-  "VideoPinballDeterministic-v3": {
+  "VideoPinballDeterministic-v4": {
     "actions": "680dc83e85ea9c0ec0bed4ba7ae3a87dbf66cc40db1922a0ec9debfca671766f",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -4181,7 +4181,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "VideoPinballNoFrameskip-v3": {
+  "VideoPinballNoFrameskip-v4": {
     "actions": "680dc83e85ea9c0ec0bed4ba7ae3a87dbf66cc40db1922a0ec9debfca671766f",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -4193,7 +4193,7 @@
     "observations": "5e3df66c036da1db3970c291cf1b004fb2fc9314b0a93d6aeca9682e50dc1c86",
     "rewards": "bab1b66cf5879d0fb2d6fa6554ff9d533f118e17996176f51b82dc5b407a8aba"
   },
-  "WizardOfWor-ram-v3": {
+  "WizardOfWor-ram-v4": {
     "actions": "b199b81b77e4e2a8aad9a5663315bd9f7a65ba9ad191c7f8645848e7291df62e",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "835164cc692c4d3431937806eaea0af9efb0ba50ed4c57664ab47bfc223273ab",
@@ -4205,7 +4205,7 @@
     "observations": "e8209a38820896418096ec5f1a4bb9f7eca99ecaca9eae64911ec57a605df7ca",
     "rewards": "ee6162f282d1303b74c9e3540debe76757fd071f2494245120b9c2275f37c022"
   },
-  "WizardOfWor-ramDeterministic-v3": {
+  "WizardOfWor-ramDeterministic-v4": {
     "actions": "b199b81b77e4e2a8aad9a5663315bd9f7a65ba9ad191c7f8645848e7291df62e",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "e8209a38820896418096ec5f1a4bb9f7eca99ecaca9eae64911ec57a605df7ca",
@@ -4217,7 +4217,7 @@
     "observations": "e115edebf3181acf8e0349a5cb69f7a1f04b38895e1abcd7c937ffcf029b7c33",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "WizardOfWor-ramNoFrameskip-v3": {
+  "WizardOfWor-ramNoFrameskip-v4": {
     "actions": "b199b81b77e4e2a8aad9a5663315bd9f7a65ba9ad191c7f8645848e7291df62e",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "e115edebf3181acf8e0349a5cb69f7a1f04b38895e1abcd7c937ffcf029b7c33",
@@ -4229,7 +4229,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "bab1b66cf5879d0fb2d6fa6554ff9d533f118e17996176f51b82dc5b407a8aba"
   },
-  "WizardOfWor-v3": {
+  "WizardOfWor-v4": {
     "actions": "b199b81b77e4e2a8aad9a5663315bd9f7a65ba9ad191c7f8645848e7291df62e",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -4241,7 +4241,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "ee6162f282d1303b74c9e3540debe76757fd071f2494245120b9c2275f37c022"
   },
-  "WizardOfWorDeterministic-v3": {
+  "WizardOfWorDeterministic-v4": {
     "actions": "b199b81b77e4e2a8aad9a5663315bd9f7a65ba9ad191c7f8645848e7291df62e",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -4253,7 +4253,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "WizardOfWorNoFrameskip-v3": {
+  "WizardOfWorNoFrameskip-v4": {
     "actions": "b199b81b77e4e2a8aad9a5663315bd9f7a65ba9ad191c7f8645848e7291df62e",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -4265,7 +4265,7 @@
     "observations": "eeecf8e7f0da2f243b6324be1e82fe575c596ac425db29fbb2d1d0a4d5a92b04",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "YarsRevenge-ram-v3": {
+  "YarsRevenge-ram-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "0a85a374ac24a6e529ace2c73c500c4c0c6e75196d17546339f18208f9c3782a",
@@ -4277,7 +4277,7 @@
     "observations": "daedd650e7740a0fd89b23087fb7725af0117de0d2270158866160a5efa9280f",
     "rewards": "245271d389eb7b4f5af151e7531543e3db6e1b875e8f3971962ff65e7ca0864c"
   },
-  "YarsRevenge-ramDeterministic-v3": {
+  "YarsRevenge-ramDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "2be6b6ffa0b8ebf02107ac62b32fbdf0eb71e9b4b1cd8413e8553fb2d372a4cd",
@@ -4289,7 +4289,7 @@
     "observations": "9da3d25eda5e56eff61fdc2ab2ea932e452b3e33f55b0e5b6f58038698c34a67",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "YarsRevenge-ramNoFrameskip-v3": {
+  "YarsRevenge-ramNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "e612f3bbed3c37a67a084d8b1e9d0347164c3049775b80e1c753b240f34b6abc",
@@ -4301,7 +4301,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "YarsRevenge-v3": {
+  "YarsRevenge-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -4313,7 +4313,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "245271d389eb7b4f5af151e7531543e3db6e1b875e8f3971962ff65e7ca0864c"
   },
-  "YarsRevengeDeterministic-v3": {
+  "YarsRevengeDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -4325,7 +4325,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "YarsRevengeNoFrameskip-v3": {
+  "YarsRevengeNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -4337,7 +4337,7 @@
     "observations": "ef8fd5dc6e04dc7d431e193a9992dcbe69ab7e92ff73f41d1cd243fa57f6ad2e",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Zaxxon-ram-v3": {
+  "Zaxxon-ram-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "3e88b1d6f877e305fa027adbfadb2d04b1704e55943420765780650cb2a654c9",
@@ -4349,7 +4349,7 @@
     "observations": "612f7497980ddbc855eb965b1ad653328068da603c4489c8b40389e6d55c79d6",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Zaxxon-ramDeterministic-v3": {
+  "Zaxxon-ramDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "5646dd21f02bab5a5102e314eddc388f7412320f36b5e205742fe9b57b17bfd7",
@@ -4361,7 +4361,7 @@
     "observations": "acf18e372aa313f98e7866cc927e194af01a84e7d15d218cb2f3bf39182e4c96",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Zaxxon-ramNoFrameskip-v3": {
+  "Zaxxon-ramNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "5e2daefd2299788423c5a04bf25b483d07e91d5179671024a29b56be41ac25b2",
@@ -4373,7 +4373,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "Zaxxon-v3": {
+  "Zaxxon-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -4385,7 +4385,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "ZaxxonDeterministic-v3": {
+  "ZaxxonDeterministic-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
@@ -4397,7 +4397,7 @@
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
     "rewards": "04db9812be236ea437cbda6cea214bba8c79760fb57a66176704503576f6f390"
   },
-  "ZaxxonNoFrameskip-v3": {
+  "ZaxxonNoFrameskip-v4": {
     "actions": "a642086826823e658c283b56dd79f14af59846af2c3d93fad08c3bc84bf3b748",
     "dones": "ecfbe8578a5aac6442d7b65f2e4bd4f6d70e5cdc76c1d6868ee031460c7477b9",
     "observations": "01bc2647e2df61bfa95036ae892f69cba51909cf6d87ab94ba8168d105358b97",
diff --git a/gym/utils/play.py b/gym/utils/play.py
index b433ac0c856..8587f6f8d5a 100644
--- a/gym/utils/play.py
+++ b/gym/utils/play.py
@@ -182,5 +182,5 @@ def callback(self, obs_t, obs_tp1, action, rew, done, info):
 
 
 if __name__ == '__main__':
-    env = gym.make("MontezumaRevengeNoFrameskip-v3")
+    env = gym.make("MontezumaRevengeNoFrameskip-v4")
     play(env, zoom=4, fps=60)
diff --git a/setup.py b/setup.py
index 64172e5394f..e7d5784bbe6 100644
--- a/setup.py
+++ b/setup.py
@@ -7,7 +7,7 @@
 
 # Environment-specific dependencies.
 extras = {
-  'atari': ['atari_py>=0.0.21', 'Pillow', 'PyOpenGL'],
+  'atari': ['atari_py>=0.0.22', 'Pillow', 'PyOpenGL'],
   'board_game' : ['pachi-py>=0.0.19'],
   'box2d': ['Box2D-kengz'],
   'classic_control': ['PyOpenGL'],