fix(training): use n-step windows during a3c training

 - the current model has a bug where it uses the entire history during training. This means that predictions progressively get slower with longer episode lengths.
 - clipping to 3-step by default because that seems to be popular, and the a3c agent seems capable of using it to quickly solve easy poly problems

Author: justindujardin

libraries/mathy_python/mathy/agents/episode_memory.py

@@ -58,9 +58,9 @@ def clear(self):
         self.grouping_changes = []
 
     def to_window_observation(
-        self, observation: MathyObservation, window_size: int = 1
+        self, observation: MathyObservation, window_size: int = 3
     ) -> MathyWindowObservation:
-        previous = -(window_size - 1)
+        previous = -(max(window_size - 1, 1))
         window_observations = self.observations[previous:] + [observation]
         return observations_to_window(window_observations)
 

libraries/mathy_python/mathy/agents/policy_value_model.py

@@ -4,7 +4,6 @@
 from pathlib import Path
 from shutil import copyfile
 from typing import Any, Callable, Dict, List, Optional, Tuple, cast
-from memory_profiler import profile
 
 import numpy as np
 import srsly
@@ -275,7 +274,7 @@ def handshake_keras(m: ThincPolicyValueModel):
         # If we're doing transfer, reset optimizer steps
         if is_main and args.init_model_from is not None:
             msg.info("reset optimizer steps to 0 for transfer model")
-            model.optimizer.iterations.assign(0)
+            model.unwrapped.optimizer.iterations.assign(0)
     elif required:
         print_error(
             ValueError("Model Not Found"),