fix(mathy_alpha_sm): revert defaults to last known good model

 - reduce Grouping Control and Entropy loss signals
 - Clip grouping control

Author: justindujardin

libraries/mathy_python/mathy/agents/a3c/config.py

@@ -32,7 +32,6 @@ class A3CConfig(BaseConfig):
     # MCTS provides higher quality observations at extra computational cost.
     mcts_sims: int = 200
 
-
     # Whether to use the grouping change aux task
     use_grouping_control = True
     # Clip signal at 0.0 so it does not optimize into the negatives
@@ -45,19 +44,19 @@ class A3CConfig(BaseConfig):
     # other threads time to process. This is useful for
     # running more threads than you have processors to
     # get a better diversity of experience.
-    worker_wait: float = 0.01
+    worker_wait: float = 0.1
 
     # The number of worker agents to create.
     num_workers: int = 3
 
     # NOTE: scaling down h_loss is observed to be important to keep it from
     #       destabilizing the overall loss when it grows very small
-    entropy_loss_scaling = 1.0
+    entropy_loss_scaling = 0.05
     # Whether to scale entropy loss so it's 0-1
     normalize_entropy_loss = True
 
     # How much to scale down loss values from auxiliary tasks
-    aux_tasks_weight_scale = 1.0
+    aux_tasks_weight_scale = 0.1
     # The lambda value for generalized lambda returns to calculate value loss
     # 0.0 = bootstrap values, 1.0 = discounted
     td_lambda: float = 0.2

libraries/mathy_python/mathy/agents/base_config.py

@@ -37,7 +37,7 @@ class Config:
     num_workers: int = 3
     # The lambda value for generalized lambda returns to calculate value loss
     # 0.0 = bootstrap values, 1.0 = discounted
-    td_lambda: float = 0.3
+    td_lambda: float = 0.2
     # Verbose setting to print out worker_0 training steps. Useful for trying
     # to find problems.
     print_training: bool = False
@@ -52,7 +52,7 @@ class Config:
     # Whether to use the grouping change aux task
     use_grouping_control = False
     # Clip signal at 0.0 so it does not optimize into the negatives
-    clip_grouping_control = False
+    clip_grouping_control = True
 
     # Include the time/type environment features in the embeddings
     use_env_features = False

libraries/mathy_python/mathy/agents/policy_value_model.py

@@ -237,7 +237,7 @@ def get_or_create_policy_model(
 
 
 def load_policy_value_model(
-    model_data_folder: str,
+    model_data_folder: str, silent: bool = False
 ) -> Tuple[PolicyValueModel, BaseConfig]:
     meta_file = Path(model_data_folder) / "model.config.json"
     if not meta_file.exists():
@@ -263,7 +263,7 @@ def load_policy_value_model(
     model.build(initial_state.to_input_shapes())
     model.predict(init_inputs)
     model.predict_next(init_inputs)
-    if args.verbose:
+    if not silent:
         with msg.loading(f"Loading model: {model_file}..."):
             _load_model(model, str(model_file), str(optimizer_file))
         msg.good(f"Loaded model: {model_file}")

libraries/mathy_python/mathy/mathy.py

@@ -17,9 +17,10 @@ def __init__(
         model_path: str = None,
         model: PolicyValueModel = None,
         config: BaseConfig = None,
+        silent: bool = False,
     ):
         if model_path is not None:
-            self.model, self.config = load_policy_value_model(model_path)
+            self.model, self.config = load_policy_value_model(model_path, silent=silent)
         elif model is not None and config is not None:
             if not isinstance(model, PolicyValueModel):
                 raise ValueError("model must derive PolicyValueModel for compatibility")