| Relu_gate -> "relu_gate"

| Cmplt -> "lt"

| Cmpne -> "le"

| Max -> "max"

| Min -> "min"

let stream = Backend.(new_stream @@ get_device ~ordinal:0) in

let ctx = Backend.make_context stream in

(* Hey is inferred to be a matrix. *)

let zero_to_twenty = TDSL.range 20 in

let y = TDSL.O.(( + ) ~label:[ "y" ] (hey * zero_to_twenty) zero_to_twenty) in

Train.set_hosted hey.value;

let%op d = (a *. b) + (b **. 3) in

let%op c = c + c + 1 in

let%op c = c + 1 + c + ~-a in

let%op e = c - d in

let%op f = e *. e in

let%op g = f /. 2 in

let moons_classes = TDSL.init_const ~l:"moons_classes" ?b ~o:[ 1 ] moons_classes in

let batch_n, bindings = IDX.get_static_symbol ~static_range:n_batches IDX.empty in

let step_n, bindings = IDX.get_static_symbol bindings in

let%op moons_input = moons_flat @| batch_n in

(* Tell shape inference to make a minibatch axis. *)

let () =

let losses = ref [] in

let log_losses = ref [] in

let learning_rates = ref [] in

(* We don't need a regression loss formula thanks to weight_decay built into the sgd_update

let scalar_loss, weight_decay =

let init_time = Time_now.nanoseconds_since_unix_epoch () in

let%op mlp x =

"w4"

in

(* TINY for debugging: *)

let start_time = ref None in

let weight_decay = 0.0002 in

Arrayjit.Schedulers.sync_suggested_num_streams := num_streams;

let steps = epochs * n_batches in

let weight_decay = 0.0002 in

let noise () = Rand.float_range (-0.1) 0.1 in

let moons_flat =

let%op moons_input = moons_flat @| batch_n in

let%op moons_class = moons_classes @| batch_n in

let%op scalar_loss = (margin_loss ++ "...|... => 0") /. !..batch_size in

let update = Train.grad_update scalar_loss in

let moons_flat ~b = TDSL.init_const ~l:"moons_flat" ~b ~o:[ 2 ] moons_flat in

let moons_classes = Array.init (len * 2) ~f:(fun i -> if i % 2 = 0 then 1. else -1.) in

let moons_classes ~b = TDSL.init_const ~l:"moons_classes" ~b ~o:[ 1 ] moons_classes in

(* let%op mlp x = "b" + ("w" * x) in *)

(* We don't need a regression loss formula thanks to weight_decay built into the sgd_update

let weight_decay = 0.0002 in

<\body>

<doc-data|<doc-title|OCANNL optimization framework>|<doc-subtitle|Tensor

shape inference, concise notation, multidevice

Stafiniak>|<\author-note>

Since April 2024, <hlink|<with|font-family|tt|<with|color|orange|a><with|color|blue|hrefs>>|https://ahrefs.com/>

</author-note>>>>

<abstract-data|<abstract|OCANNL is a Deep Learning framework with

fetaures <with|font-shape|italic|parameter punning> (strings become

let-bindings of tensors) and inline output dimensions specification. Full

example of a Multi Layer Perceptron with 2 hidden layers and Rectified

<verbatim|w1>, <verbatim|b2>, <verbatim|w2>, <verbatim|b3>, <verbatim|w3>,

and a tensor-returning function <verbatim|mlp>:

<\verbatim-code>

let%op mlp x =

x)))))

</verbatim-code>

type mlp_layer_config = { label : string list; hid_dim : int }

type mlp_config = { label : string list; hid_dims : int list }

let module NTDSL = Initial_NTDSL in

let%cd op_asn ~v ~t1 ~projections = v =: relu v1 ~projections in

let%cd grad_asn ~v ~g ~t1 ~projections = g1 =+ v -?/ g in

module NDO_without_pow = struct

let ( * ) = matmul ~grad_spec:Prohibit_grad

let ( *. ) = pointmul ~grad_spec:Prohibit_grad

let ( + ) = add ~grad_spec:Prohibit_grad

let ( !. ) = Tensor.number ~grad_spec:Prohibit_grad

let ( !.. ) ?label i = Tensor.number ?label ~grad_spec:Prohibit_grad @@ Float.of_int i

let ( - ) = sub ~grad_spec:Prohibit_grad

let ( *. ) = pointmul ~grad_spec:If_needed

let ( + ) = add ~grad_spec:If_needed

let ( **. ) ?label base exp = pointpow ?label exp base ~grad_spec:If_needed

let ( !. ) = Tensor.number ~grad_spec:If_needed

let ( !.. ) ?label i = Tensor.number ?label ~grad_spec:If_needed @@ Float.of_int i

let ( !@ ) = embed_symbol