Use spike decay for unet

Even 1e-4 spiked the KL

Author: sgoodfriend

rl_algo_impls/hyperparams/ppo.yml

@@ -314,7 +314,8 @@ unet-MicrortsDefeatRandomEnemySparseReward-v3:
     v_hidden_sizes: [256, 128]
   algo_hyperparams:
     <<: *microrts-ai-algo-defaults
-    learning_rate: !!float 1e-4
+    learning_rate: !!float 2.5e-4
+    learning_rate_decay: spike
 
 MicrortsDefeatCoacAIShaped-v3: &microrts-coacai-defaults
   <<: *microrts-ai-defaults
@@ -356,7 +357,8 @@ unet-MicrortsDefeatCoacAIShaped-v3-diverseBots:
     v_hidden_sizes: [256, 128]
   algo_hyperparams:
     <<: *microrts-ai-algo-defaults
-    learning_rate: !!float 1e-4
+    learning_rate: !!float 2.5e-4
+    learning_rate_decay: spike
 
 HalfCheetahBulletEnv-v0: &pybullet-defaults
   n_timesteps: !!float 2e6