Use a fixed bias 0.5 in the half-moons examples

lukstafi · lukstafi · commit 5077531947b2 · 2025-07-31T13:23:01.000+02:00
(1) current randomness doesn't work with sizes not divisible by 2 / 4 / 8 / 16 (double / single / half / fp8);
(2) learnable bias is redundant as the other weights can adapt
(3) 0 bias does not train well with ReLU activations
diff --git a/bin/micrograd_demo.ml b/bin/micrograd_demo.ml
@@ -25,7 +25,7 @@ let experiment seed ~no_batch_shape_inference ~use_builtin_weight_decay () =
   let moons_classes = TDSL.rebatch ~l:"moons_classes" moons_classes_ndarray () in
   let batch_n, bindings = IDX.get_static_symbol ~static_range:n_batches IDX.empty in
   let step_n, bindings = IDX.get_static_symbol bindings in
-  let%op mlp x = "b3" + ("w3" * relu ("b2" hid_dim + ("w2" * relu ("b1" hid_dim + ("w1" * x))))) in
+  let%op mlp x = "w3" * relu ("b2" hid_dim + ("w2" * relu ("b1" hid_dim + ("w1" * x)))) in
   let%op moons_input = moons_flat @| batch_n in
   (* Tell shape inference to make a minibatch axis. *)
   let () =
@@ -51,9 +51,7 @@ let experiment seed ~no_batch_shape_inference ~use_builtin_weight_decay () =
       (scalar_loss, 0.0002)
     else
       let%op ssq w = (w **. 2) ++ "...|...->... => 0" in
-      let reg_loss =
-        List.map ~f:ssq [ w1; w2; w3; b1; b2; b3 ] |> List.reduce_exn ~f:TDSL.O.( + )
-      in
+      let reg_loss = List.map ~f:ssq [ w1; w2; w3; b1; b2 ] |> List.reduce_exn ~f:TDSL.O.( + ) in
       let%op scalar_loss =
         ((margin_loss ++ "...|... => 0") /. !..batch_size) + (0.0001 *. reg_loss)
       in
diff --git a/bin/moons_demo.ml b/bin/moons_demo.ml
@@ -23,7 +23,7 @@ let demo () =
   let steps = epochs * n_batches in
   let weight_decay = 0.0002 in
 
-  let%op mlp x = "b3" + ("w3" * relu ("b2" hid_dim + ("w2" * relu ("b1" hid_dim + ("w1" * x))))) in
+  let%op mlp x = "w3" * relu ("b2" hid_dim + ("w2" * relu ("b1" hid_dim + ("w1" * x)))) in
 
   let config = Datasets.Half_moons.Config.{ noise_range = 0.1; seed = Some seed } in
   let moons_coordinates, moons_labels = Datasets.Half_moons.generate_single_prec ~config ~len () in
diff --git a/bin/moons_demo_parallel.ml b/bin/moons_demo_parallel.ml
@@ -23,8 +23,8 @@ let experiment ~seed ~backend_name ~config () =
   let moons_classes_ndarray = Ir.Ndarray.as_array Ir.Ops.Double moons_labels in
   let moons_flat = TDSL.rebatch ~l:"moons_flat" moons_flat_ndarray () in
   let moons_classes = TDSL.rebatch ~l:"moons_classes" moons_classes_ndarray () in
-  let%op mlp x = "b3" + ("w3" * relu ("b2" hid_dim + ("w2" * relu ("b1" hid_dim + ("w1" * x))))) in
-  (* let%op mlp x = "b" + ("w" * x) in *)
+  let%op mlp x = "w3" * relu ("b2" hid_dim + ("w2" * relu ("b1" hid_dim + ("w1" * x)))) in
+  (* let%op mlp x = ("w" * x) in *)
   let%op loss_fn ~output ~expectation = relu (!..1 - (expectation *. output)) in
   (* We don't need a regression loss formula thanks to weight_decay built into the sgd_update
      computation. *)
diff --git a/test/einsum/moons_demo_variant.ml b/test/einsum/moons_demo_variant.ml
@@ -27,7 +27,7 @@ let () =
   let step_n, bindings = IDX.get_static_symbol bindings in
   let moons_flat = TDSL.rebatch ~l:"moons_flat" moons_flat_ndarray () in
   let moons_classes = TDSL.rebatch ~l:"moons_classes" moons_classes_ndarray () in
-  let%op mlp x = "b3" + ("w3" * relu ("b2" 16 + ("w2" * relu ("b1" 16 + ("w1" * x))))) in
+  let%op mlp x = 0.5 + ("w3" * relu ("b2" 16 + ("w2" * relu ("b1" 16 + ("w1" * x))))) in
   (* Don't decay the learning rate too quickly, it behaves better than in the original. *)
   let%op moons_input = moons_flat @| batch_n in
   (* THIS IS THE SPECIFIC SHAPE INFERENCE ASPECT OF THE TEST. *)
diff --git a/test/training/moons_demo.ml b/test/training/moons_demo.ml
@@ -16,7 +16,7 @@ let main () =
   let len = 200 in
   let batch_size = 10 in
   let n_batches = 2 * len / batch_size in
-  let epochs = 10 in
+  let epochs = 50 in
   let steps = epochs * 2 * len / batch_size in
   let moons_config = Datasets.Half_moons.Config.{ noise_range = 0.1; seed = Some 5 } in
   let moons_coordinates, moons_labels = Datasets.Half_moons.generate ~config:moons_config ~len () in
@@ -26,7 +26,7 @@ let main () =
   let step_n, bindings = IDX.get_static_symbol bindings in
   let moons_flat = TDSL.rebatch ~l:"moons_flat" moons_flat_ndarray () in
   let moons_classes = TDSL.rebatch ~l:"moons_classes" moons_classes_ndarray () in
-  let%op mlp x = "b3" + ("w3" * relu ("b2" 16 + ("w2" * relu ("b1" 16 + ("w1" * x))))) in
+  let%op mlp x = 0.5 + ("w3" * relu ("b2" 16 + ("w2" * relu ("b1" 16 + ("w1" * x))))) in
   (* Don't decay the learning rate too quickly, it behaves better than in the original. *)
   let%op moons_input = moons_flat @| batch_n in
   let%op moons_class = moons_classes @| batch_n in
diff --git a/test/training/moons_demo_parallel.ml b/test/training/moons_demo_parallel.ml
@@ -18,15 +18,15 @@ let main () =
   let len = batch_size * 20 in
   let init_lr = 0.1 in
   (* let epochs = 10 in *)
-  let epochs = 20 in
+  let epochs = 100 in
   (* let epochs = 1 in *)
   let moons_config = Datasets.Half_moons.Config.{ noise_range = 0.1; seed = Some seed } in
   let moons_coordinates, moons_labels = Datasets.Half_moons.generate ~config:moons_config ~len () in
   let moons_flat_ndarray = Ir.Ndarray.as_array Ir.Ops.Double moons_coordinates in
   let moons_classes_ndarray = Ir.Ndarray.as_array Ir.Ops.Double moons_labels in
   let moons_flat = TDSL.rebatch ~l:"moons_flat" moons_flat_ndarray () in
   let moons_classes = TDSL.rebatch ~l:"moons_classes" moons_classes_ndarray () in
-  let%op mlp x = "b3" + ("w3" * relu ("b2" hid_dim + ("w2" * relu ("b1" hid_dim + ("w1" * x))))) in
+  let%op mlp x = 0.5 + ("w3" * relu ("b2" hid_dim + ("w2" * relu ("b1" hid_dim + ("w1" * x))))) in
   (* let%op mlp x = "b" + ("w" * x) in *)
   let%op loss_fn ~output ~expectation = relu (!..1 - (expectation *. output)) in
   (* We don't need a regression loss formula thanks to weight_decay built into the sgd_update
