From 2406f5c6f3ab98cf83395e9867976874f12d4805 Mon Sep 17 00:00:00 2001 From: Jon Ludlam Date: Thu, 30 Jul 2015 17:07:35 +0000 Subject: [PATCH 1/7] Add stringext Signed-off-by: Jon Ludlam --- _oasis | 2 +- _tags | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/_oasis b/_oasis index ed37c80..f3b06e1 100644 --- a/_oasis +++ b/_oasis @@ -30,7 +30,7 @@ Executable "xenvmd" MainIs: xenvmd.ml Custom: true Install: false - BuildDepends: lwt, lwt.unix, mirage-block-unix, mirage-clock-unix, devmapper, threads, lvm, cstruct, oUnit, io-page, io-page.unix, cmdliner, sexplib.syntax, xenvmidl, lvm, lvm.mapper, shared-block-ring + BuildDepends: lwt, lwt.unix, mirage-block-unix, mirage-clock-unix, devmapper, threads, lvm, cstruct, oUnit, io-page, io-page.unix, stringext, cmdliner, sexplib.syntax, xenvmidl, lvm, lvm.mapper, shared-block-ring Executable "xenvm" ByteOpt: -warn-error +a diff --git a/_tags b/_tags index 2ca2171..51125fb 100644 --- a/_tags +++ b/_tags @@ -1,5 +1,5 @@ # OASIS_START -# DO NOT EDIT (digest: ff372a3fb61a268f33926cbcbfd6c5ab) +# DO NOT EDIT (digest: 2190fe0a8a00a6b8cc32c1b1dff4f3d1) # Ignore VCS directories, you can use the same kind of rule outside # OASIS_START/STOP if you want to exclude directories that contains # useless stuff for the build process @@ -52,6 +52,7 @@ true: annot, bin_annot : pkg_sexplib : pkg_sexplib.syntax : pkg_shared-block-ring +: pkg_stringext : pkg_threads : use_xenvmidl : pkg_cmdliner @@ -72,6 +73,7 @@ true: annot, bin_annot : pkg_sexplib : pkg_sexplib.syntax : pkg_shared-block-ring +: pkg_stringext : pkg_threads : use_xenvmidl : custom From 455c9078426ad8d32ca17cd691687c2d26c61910 Mon Sep 17 00:00:00 2001 From: Jon Ludlam Date: Thu, 30 Jul 2015 16:35:53 +0000 Subject: [PATCH 2/7] Persist which hosts have connected via LVM tags Signed-off-by: Jon Ludlam --- xenvmd/xenvmd.ml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/xenvmd/xenvmd.ml b/xenvmd/xenvmd.ml index e79a56e..4d689fd 100644 --- a/xenvmd/xenvmd.ml +++ b/xenvmd/xenvmd.ml @@ -291,6 +291,11 @@ module VolumeManager = struct end else begin match Vg_IO.find vg toLVM, Vg_IO.find vg fromLVM, Vg_IO.find vg freeLVM with | Some toLVM_id, Some fromLVM_id, Some freeLVM_id -> + (* Persist at this point that we're going to connect this host *) + (* All of the following logic is idempotent *) + write (fun vg -> + Lvm.Vg.add_tag vg toLVM "xenvm_connected" + ) >>= fun () -> Hashtbl.replace host_connections name Xenvm_interface.Resuming_to_LVM; let background_t () = Vg_IO.Volume.connect toLVM_id @@ -305,7 +310,6 @@ module VolumeManager = struct ToLVM.resume toLVM_q >>= fun () -> - Vg_IO.Volume.connect fromLVM_id >>= function | `Error _ -> fail (Failure (Printf.sprintf "Failed to open %s" fromLVM)) @@ -394,7 +398,7 @@ module VolumeManager = struct let to_lvm = List.assoc name !to_LVMs in debug "Suspending ToLVM queue for %s" name; ToLVM.suspend to_lvm - >>= fun () -> + >>= fun () -> (* There may still be updates in the ToLVM queue *) Lwt_mutex.with_lock flush_m (fun () -> flush_already_locked name) >>= fun () -> @@ -402,6 +406,10 @@ module VolumeManager = struct to_LVMs := List.filter (fun (n, _) -> n <> name) !to_LVMs; from_LVMs := List.filter (fun (n, _) -> n <> name) !from_LVMs; free_LVs := List.filter (fun (n, _) -> n <> name) !free_LVs; + let toLVM = toLVM name in + write (fun vg -> + Lvm.Vg.remove_tag vg toLVM "xenvm_connected" + ) >>= fun () -> Hashtbl.remove host_connections name; return () | x -> From b1d5de225e38ec78a606b4508f7a1a05708511a6 Mon Sep 17 00:00:00 2001 From: Jon Ludlam Date: Thu, 30 Jul 2015 17:07:48 +0000 Subject: [PATCH 3/7] Reconnect to hosts after a restart Signed-off-by: Jon Ludlam --- xenvmd/xenvmd.ml | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/xenvmd/xenvmd.ml b/xenvmd/xenvmd.ml index 4d689fd..a0fb61f 100644 --- a/xenvmd/xenvmd.ml +++ b/xenvmd/xenvmd.ml @@ -31,6 +31,8 @@ let fatal_error_t msg = error "%s" msg; fail (Failure msg) +let connected_tag = "xenvm_connected" + let fatal_error msg m = m >>= function | `Error (`Msg x) -> fatal_error_t (msg ^ ": " ^ x) | `Error `Suspended -> fatal_error_t (msg ^ ": queue is suspended") @@ -294,7 +296,7 @@ module VolumeManager = struct (* Persist at this point that we're going to connect this host *) (* All of the following logic is idempotent *) write (fun vg -> - Lvm.Vg.add_tag vg toLVM "xenvm_connected" + Lvm.Vg.add_tag vg toLVM connected_tag ) >>= fun () -> Hashtbl.replace host_connections name Xenvm_interface.Resuming_to_LVM; let background_t () = @@ -408,7 +410,7 @@ module VolumeManager = struct free_LVs := List.filter (fun (n, _) -> n <> name) !free_LVs; let toLVM = toLVM name in write (fun vg -> - Lvm.Vg.remove_tag vg toLVM "xenvm_connected" + Lvm.Vg.remove_tag vg toLVM connected_tag ) >>= fun () -> Hashtbl.remove host_connections name; return () @@ -463,6 +465,30 @@ module VolumeManager = struct else None in return { Xenvm_interface.name; connection_state; fromLVM; toLVM; freeExtents } ) !to_LVMs + + let reconnect_all () = + read (fun vg -> + debug "Reconnecting"; + Lvm.Vg.LVs.fold (fun key v acc -> + debug "Checking LV: %s" v.Lvm.Lv.name; + let name = v.Lvm.Lv.name in + match Stringext.split name ~on:'-' |> List.rev with + | "toLVM" :: host_bits -> + let host = String.concat "-" (List.rev host_bits) in + debug "This is a 'toLVM' LV"; + (* It's a toLVM - check to see whether it has the 'connected' flag *) + let tags = List.map Lvm.Name.Tag.to_string v.Lvm.Lv.tags in + let connected = List.mem connected_tag tags in + debug "host=%s connected=%b" host connected; + (host,connected)::acc + | e -> + debug "got list: %s" (String.concat "," e); + acc) + vg.Lvm.Vg.lvs [] |> Lwt.return + ) >>= fun host_states -> + Lwt_list.iter_s (fun (host, connected) -> + if connected then connect host else disconnect host) host_states + end let flush_all () = @@ -748,7 +774,9 @@ let run port sock_path config = >>= fun () -> FreePool.start Xenvm_interface._journal_name >>= fun () -> - + VolumeManager.Host.reconnect_all () + >>= fun () -> + let rec service_queues () = (* 0. Have any local allocators restarted? *) FreePool.resend_free_volumes config From 0484a40b15d4ae4befddbf6289b4ca643930c986 Mon Sep 17 00:00:00 2001 From: Jon Ludlam Date: Fri, 31 Jul 2015 14:11:18 +0000 Subject: [PATCH 4/7] Make code more readable, following review from Si Beaumont. Signed-off-by: Jon Ludlam --- xenvmd/xenvmd.ml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/xenvmd/xenvmd.ml b/xenvmd/xenvmd.ml index a0fb61f..d08be5e 100644 --- a/xenvmd/xenvmd.ml +++ b/xenvmd/xenvmd.ml @@ -478,16 +478,16 @@ module VolumeManager = struct debug "This is a 'toLVM' LV"; (* It's a toLVM - check to see whether it has the 'connected' flag *) let tags = List.map Lvm.Name.Tag.to_string v.Lvm.Lv.tags in - let connected = List.mem connected_tag tags in - debug "host=%s connected=%b" host connected; - (host,connected)::acc + let was_connected = List.mem connected_tag tags in + debug "host=%s was_connected=%b" host was_connected; + (host,was_connected)::acc | e -> debug "got list: %s" (String.concat "," e); acc) vg.Lvm.Vg.lvs [] |> Lwt.return ) >>= fun host_states -> - Lwt_list.iter_s (fun (host, connected) -> - if connected then connect host else disconnect host) host_states + Lwt_list.iter_s (fun (host, was_connected) -> + if was_connected then connect host else disconnect host) host_states end From 18907163c71ffb7b19cd7602c980484d3c707be6 Mon Sep 17 00:00:00 2001 From: Jon Ludlam Date: Fri, 31 Jul 2015 14:17:37 +0000 Subject: [PATCH 5/7] Add stringext to opam file Signed-off-by: Jon Ludlam --- opam | 1 + 1 file changed, 1 insertion(+) diff --git a/opam b/opam index 98309a0..384ba42 100644 --- a/opam +++ b/opam @@ -27,4 +27,5 @@ depends: [ "ounit" "oasis" "ocveralls" + "stringext" ] From f90f9cd473cf79db6038eb03adef60659571e9a9 Mon Sep 17 00:00:00 2001 From: Jon Ludlam Date: Fri, 31 Jul 2015 15:10:59 +0000 Subject: [PATCH 6/7] Add a test for reconnect Signed-off-by: Jon Ludlam --- cleanup.sh | 2 +- setup.sh | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/cleanup.sh b/cleanup.sh index ef38313..a5b6f21 100755 --- a/cleanup.sh +++ b/cleanup.sh @@ -22,4 +22,4 @@ if [ "$USE_MOCK" -eq "0" ]; then losetup -d $LOOP fi -rm -f localJournal bigdisk *.out djstest-* dm-mock +rm -f localJournal bigdisk *.out* djstest-* dm-mock diff --git a/setup.sh b/setup.sh index ce6cc11..c10fa14 100755 --- a/setup.sh +++ b/setup.sh @@ -69,14 +69,20 @@ cat test.local_allocator.conf.in | sed -r "s|@BIGDISK@|$LOOP|g" | sed -r "s|@HO sleep 30 ./xenvm.native host-list /dev/djstest --configdir /tmp/xenvm.d $MOCK_ARG +# Let's check that xenvmd retains its list of connected hosts over a restart +./xenvm.native host-list /dev/djstest --configdir /tmp/xenvm.d $MOCK_ARG > host-list.out +kill `cat /tmp/xenvmd.lock` +./xenvmd.native --config ./test.xenvmd.conf > xenvmd.log.2 & +sleep 10 +./xenvm.native host-list /dev/djstest --configdir /tmp/xenvm.d $MOCK_ARG > host-list.out2 +diff host-list.out host-list.out2 + # destroy hosts ./xenvm.native host-disconnect /dev/djstest host2 --configdir /tmp/xenvm.d $MOCK_ARG ./xenvm.native host-destroy /dev/djstest host2 --configdir /tmp/xenvm.d $MOCK_ARG ./xenvm.native host-disconnect /dev/djstest host1 --configdir /tmp/xenvm.d $MOCK_ARG ./xenvm.native host-destroy /dev/djstest host1 --configdir /tmp/xenvm.d $MOCK_ARG -./xenvm.native host-list /dev/djstest --configdir /tmp/xenvm.d $MOCK_ARG - #shutdown ./xenvm.native lvchange -an /dev/djstest/live --configdir /tmp/xenvm.d $MOCK_ARG || true ./xenvm.native shutdown /dev/djstest --configdir /tmp/xenvm.d $MOCK_ARG From c375d357c799928a544d20c639899aad91cdec74 Mon Sep 17 00:00:00 2001 From: Jon Ludlam Date: Fri, 31 Jul 2015 16:51:16 +0000 Subject: [PATCH 7/7] Sort the output of host-list to ensure order is preserved. Also, diff -u to make reading the output easier. Signed-off-by: Jon Ludlam --- setup.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.sh b/setup.sh index c10fa14..f222efa 100755 --- a/setup.sh +++ b/setup.sh @@ -70,12 +70,12 @@ sleep 30 ./xenvm.native host-list /dev/djstest --configdir /tmp/xenvm.d $MOCK_ARG # Let's check that xenvmd retains its list of connected hosts over a restart -./xenvm.native host-list /dev/djstest --configdir /tmp/xenvm.d $MOCK_ARG > host-list.out +./xenvm.native host-list /dev/djstest --configdir /tmp/xenvm.d $MOCK_ARG | sort > host-list.out kill `cat /tmp/xenvmd.lock` ./xenvmd.native --config ./test.xenvmd.conf > xenvmd.log.2 & sleep 10 -./xenvm.native host-list /dev/djstest --configdir /tmp/xenvm.d $MOCK_ARG > host-list.out2 -diff host-list.out host-list.out2 +./xenvm.native host-list /dev/djstest --configdir /tmp/xenvm.d $MOCK_ARG | sort > host-list.out2 +diff -u host-list.out host-list.out2 # destroy hosts ./xenvm.native host-disconnect /dev/djstest host2 --configdir /tmp/xenvm.d $MOCK_ARG