Merge pull request #10 from ehirdoy/addinference

Execute "inference"(test) with a trained model

src/core/actor_param_client.ml

@@ -68,11 +68,8 @@ module Make
 
     try%lwt (
       (* start client service *)
-      let thread_0 = Net.listen addr (process context) in
-      let thread_1 = heartbeat context in
-      let%lwt () = thread_0 in
-      let%lwt () = thread_1 in
-      Lwt.return ()
+      let%lwt () = Net.listen addr (process context) in
+      heartbeat context
     )
     with _ -> (
       (* clean up when client exits *)

src/core/actor_param_server.ml

@@ -95,11 +95,8 @@ module Make
 
     try%lwt (
       (* start server service *)
-      let thread_0 = Net.listen context.my_addr (process context) in
-      let thread_1 = heartbeat context in
-      let%lwt () = thread_0 in
-      let%lwt () = thread_1 in
-      Lwt.return ()
+      let%lwt () = Net.listen context.my_addr (process context) in
+      heartbeat context
     )
     with _ -> (
       (* clean up when server exits *)

test/mirage/TODO.md

@@ -0,0 +1,32 @@
+# ToDo
+
+## Bugfix
+- Don't kill when it's done.
+- Lwt'ization of loading data from files. Potentially fail.
+
+## New features
+- report: add memory usage instrument by Gc module
+  - memory/execution time profiling tools: just interested to briefly discuss what kind of tools being used for profiling. Current I'm using valgrind but not very handy.
+- add support of dynamic population of workers
+  - Depends on barrier method used. E.g:
+    - the management of workers’ information in the book.
+      - if it’s the BSP, then removing a worker will stop all the others;
+      - if it’s ASP, then dynamically removing one or adding it back should be no problem.
+- evaluation setup: use multiple rpi devices, or one large server, to scale up.
+- github: add Travis-CI for D-MNIST with Unix && MirageOS, make sure to not break with further commits
+- lwae: fix transparent parallelism behind LwAE from consistent training interface as the original AE does
+- lwae: add other Barrier support, Jianxin?
+  - for the barrier support, I've added some initial implementation here: https://github.com/jzstark/light_actor/blob/lda/src/core/actor_barrier_ssp.ml
+
+## Unsorted
+- besides mnist training, find several IoT-based use cases where multiple instances interact with each other
+- difficulties in wrapping up an Owl script into Unikernel; possible automation of this process and deployment to edge devices
+
+## Pending
+- owl: build Owl without Unix dependency for MirageOS https://github.com/owlbarn/owl/pull/372 No good idea for now
+  - ex: gettimeofday -> ptime???
+
+## Done
+- ~~mnist: add inference/test after training~~
+- ~~mnist: add Random'ization of get_next, currently it's incremented sequentially, but should be random'ized.~~
+- ~~report: add support arbitral number of clients automatically 128~~

test/mirage/config.ml

@@ -31,6 +31,7 @@ let main =
     package "actor_mirage";
     package "duration";
     package "lwt_ppx";
+    package "randomconv";
   ] in
   let keys = List.map Key.abstract [
       server_ip;
@@ -40,12 +41,12 @@ let main =
       port;
     ] in
   let keys = (Key.abstract nworkers) :: keys in
-  foreign ~packages ~keys "Unikernel.Main" (stackv4 @-> kv_ro @-> job)
+  foreign ~packages ~keys "Unikernel.Main" (stackv4 @-> kv_ro @-> random @-> job)
 
 let disk = generic_kv_ro "t"
 
 let () =
   let stack = generic_stackv4 default_network in
   register "lwae" [
-    main $ stack $ disk
+    main $ stack $ disk $ default_random
   ]

test/mirage/runall.ml

@@ -40,7 +40,7 @@ let _shall_paint c s =
 
 let color_of = function
   | Server -> colors.(0)
-  | Worker x -> colors.(Random.int (Array.length colors))
+  | Worker x -> colors.((x + 1) mod (Array.length colors))
 
 let pfx_of obj =
   _shall_paint (color_of obj) (uuid_of obj)
@@ -49,7 +49,7 @@ let cmdstr_of obj =
   let i = idx_of obj in
   let uuid = uuid_of obj in
   Printf.sprintf
-    "sudo ./solo5-hvt --disk=fat_block1.img --net=tap%d lwae.hvt --server_ip=192.168.0.1 --ipv4=192.168.0.%d/24 --ip=192.168.0.%d --port=600%d --uuid=%s --nworkers=%d"
+    "sudo ./solo5-hvt --mem=256 --disk=fat_block1.img --net=tap%d lwae.hvt --server_ip=192.168.0.1 --ipv4=192.168.0.%d/24 --ip=192.168.0.%d --port=600%d --uuid=%s --nworkers=%d"
     i i i i uuid nworkers
 
 let proc obj =

test/mirage/split-mnist-data

@@ -1,27 +1,33 @@
 #!/bin/bash
 
-img_file="train-images-idx3-ubyte"
-lvl_file="train-labels-idx1-ubyte"
-
-[ -f "$img_file" ] || curl -o - http://yann.lecun.com/exdb/mnist/"$img_file".gz > "$img_file"
-[ -f "$lvl_file" ] || curl -o - http://yann.lecun.com/exdb/mnist/"$lvl_file".gz > "$lvl_file"
-
-batch=1000
-max=$((60000 / batch - 1))
+#    bytes      file name
+#----------------------------------
+# 47040016 train-images-idx3-ubyte
+#    60008 train-labels-idx1-ubyte
+#  7840016 t10k-images-idx3-ubyte
+#    10008 t10k-labels-idx1-ubyte
 
 mkdir -p t
 
-img_header=16
-time for x in $(seq 0 $max); do
-    num=$(printf "%03d" $x)
-    unit=$((28 * 28 * batch))
-    dd if=$img_file of=t/$img_file-$num.bmp bs=1 count=$unit skip=$((img_header + x * unit))
+for t in "train" "t10k"; do
+    for x in "images" "labels"; do
+        [ "$x" = "images" ] && n=3 || n=1
+        f="$t-$x-idx$n-ubyte"
+        [ -f "$f" ] || \
+            curl -o - http://yann.lecun.com/exdb/mnist/"$f".gz | gunzip > "$f"
+    done
 done
 
-lvl_header=8
-for x in $(seq 0 $max); do
-    num=$(printf "%03d" $x)
-    unit=$((1 * batch))
-    dd if=$lvl_file of=t/$lvl_file-$num.lvl bs=1 count=$unit skip=$((lvl_header + x * unit))
+batch=1000
+ls *-ubyte | while read f; do
+    echo $f | grep -q "images" && header=16 || header=8
+    echo $f | grep -q "images" && ext="bmp" || ext="lvl"
+    echo $f | grep -q "images" && unit=$((28 * 28 * batch)) || unit=$((1 * batch))
+    echo $f | grep -q "train" && entries=60000 || entries=10000
+    max=$((entries / batch - 1))
+    for i in $(seq 0 $max); do
+        n=$(printf "%03d" $i)
+        skip=$((header + i * unit))
+        dd if="$f" of="t/$f-$n.$ext" bs=1 count="$unit" skip="$skip"
+    done
 done
-