Tutorial: Algorithmic Differentiation

Algorithmic differentiation is also known as automatic differentiation. It is a powerful tool in many fields, especially useful for fast prototyping in machine learning research. A couple of weeks ago, I started building a simple algorithmic differentiation module atop of Owl.

Currently I have implemented the prototype of forward mode. E.g., the following code first defines a function f0, then calculates from the first to the fourth derivative by calling Algodiff.Forward.derivative function.

open Algodiff.Forward

let mat_to_t_array x =
  Dense.Real.to_array x |> Array.map (fun a -> [|Float a|])
;;

let t_array_to_mat x =
  let y = Array.map (function Float a -> a) x in
  Dense.Real.of_array y 1 (Array.length y)
;;

let f0 x = Maths.(tanh x.(0));;
let f1 = derivative f0;;
let f2 = derivative f1;;
let f3 = derivative f2;;
let f4 = derivative f3;;
let x = Dense.Real.linspace (-4.) 4. 200;;
let x0 = mat_to_t_array x;;
let y0 = Array.map f0 x0 |> t_array_to_mat;;
let y1 = Array.map f1 x0 |> t_array_to_mat;;
let y2 = Array.map f2 x0 |> t_array_to_mat;;
let y3 = Array.map f3 x0 |> t_array_to_mat;;
let y4 = Array.map f4 x0 |> t_array_to_mat;;
let h = Plot.create "plot_021.png" in
Plot.set_foreground_color h 0 0 0;
Plot.set_background_color h 255 255 255;
Plot.plot ~h x y0;
Plot.plot ~h x y1;
Plot.plot ~h x y2;
Plot.plot ~h x y3;
Plot.plot ~h x y4;
Plot.output h;;

Start your utop, then load and open Owl library. Copy and past the code above, the generated figure will look like this.

In the following weeks, I will keep polishing the module and try to overload more useful mathematical operators.

Another example

let f0 x = Maths.(
  let y = exp (neg x.(0)) in
  (Float 1. -. y) /. (Float 1. +. y)
);;
let f1 = derivative f0;;
let f2 = derivative f1;;
let f3 = derivative f2;;
let f4 = derivative f3;;
let f5 = derivative f4;;
let f6 = derivative f5;;
let x = Dense.Real.linspace (-7.) 7. 200;;
let x0 = mat_to_t_array x;;
let y0 = Array.map f0 x0 |> t_array_to_mat;;
let y1 = Array.map f1 x0 |> t_array_to_mat;;
let y2 = Array.map f2 x0 |> t_array_to_mat;;
let y3 = Array.map f3 x0 |> t_array_to_mat;;
let y4 = Array.map f4 x0 |> t_array_to_mat;;
let y5 = Array.map f5 x0 |> t_array_to_mat;;
let y6 = Array.map f6 x0 |> t_array_to_mat;;
let h = Plot.create "plot_021.png" in
Plot.set_foreground_color h 0 0 0;
Plot.set_background_color h 255 255 255;
Plot.plot ~h x y0;
Plot.plot ~h x y1;
Plot.plot ~h x y2;
Plot.plot ~h x y3;
Plot.plot ~h x y4;
Plot.plot ~h x y5;
Plot.plot ~h x y6;
Plot.output h;;