From 23a06588aa69810d9e0a808df3fe36cfb3e246f6 Mon Sep 17 00:00:00 2001 From: Siraaj Khandkar Date: Sat, 22 Feb 2014 13:43:41 -0500 Subject: [PATCH 1/4] Lay down build skeleton. --- .gitignore | 3 ++ Makefile | 49 ++++++++++++++++++++++++++++++ build.ocp | 11 +++++++ src/riak_snaps/riak_snaps_main.ml | 2 ++ src/riak_snaps/riak_snaps_main.mli | 0 5 files changed, 65 insertions(+) create mode 100644 Makefile create mode 100644 build.ocp create mode 100644 src/riak_snaps/riak_snaps_main.ml create mode 100644 src/riak_snaps/riak_snaps_main.mli diff --git a/.gitignore b/.gitignore index 8fce603..80e39f7 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,4 @@ +_obuild/ +bin/ data/ +ocp-build.root* diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..d60eac2 --- /dev/null +++ b/Makefile @@ -0,0 +1,49 @@ +PROGRAMS := \ + riak-snaps + +DIR_BUILD := _obuild + + +# TODO: Test a cross-platform way of grabbing number of CPUs +MAX_BUILD_WORKERS := $(shell sysctl -n hw.ncpu) + + +.PHONY:\ + build \ + clean \ + programs \ + purge + + +programs: build bin + @for p in $(PROGRAMS); do \ + src="$(DIR_BUILD)/$$p/$$p.asm" ; \ + dst="bin/$$p" ; \ + cp $$src $$dst ; \ + done + +bin: + @mkdir -p bin + +build: ocp-build.root + @ocp-build build -njobs $(MAX_BUILD_WORKERS) + +ocp-build.root: + @ocp-build -init -njobs $(MAX_BUILD_WORKERS) + +clean: clean_bin + @ocp-build clean + @rm -f ocp-build.root* + +clean_manually: clean_bin + @rm -rf $(DIR_BUILD) + @find \ + . \ + -name '*.o' \ + -or -name '*.cmi' \ + -or -name '*.cmo' \ + -or -name '*.cmx' \ + | xargs rm -f + +clean_bin: + @rm -rf bin diff --git a/build.ocp b/build.ocp new file mode 100644 index 0000000..9e13460 --- /dev/null +++ b/build.ocp @@ -0,0 +1,11 @@ +sort = true +cflags = "-w +A" + +begin program "riak-snaps" + has_byte = false + has_asm = true + files = [ + "src/riak_snaps/riak_snaps_main.mli" + "src/riak_snaps/riak_snaps_main.ml" + ] +end diff --git a/src/riak_snaps/riak_snaps_main.ml b/src/riak_snaps/riak_snaps_main.ml new file mode 100644 index 0000000..e569e05 --- /dev/null +++ b/src/riak_snaps/riak_snaps_main.ml @@ -0,0 +1,2 @@ +let () = + print_endline "Hi" diff --git a/src/riak_snaps/riak_snaps_main.mli b/src/riak_snaps/riak_snaps_main.mli new file mode 100644 index 0000000..e69de29 From 9ab04883dd6151a0998648992c44757133814cf9 Mon Sep 17 00:00:00 2001 From: Siraaj Khandkar Date: Sat, 22 Feb 2014 14:44:17 -0500 Subject: [PATCH 2/4] Add Process module. --- build.ocp | 15 ++++++++-- src/process/process.ml | 62 +++++++++++++++++++++++++++++++++++++++++ src/process/process.mli | 16 +++++++++++ 3 files changed, 91 insertions(+), 2 deletions(-) create mode 100644 src/process/process.ml create mode 100644 src/process/process.mli diff --git a/build.ocp b/build.ocp index 9e13460..f7c5897 100644 --- a/build.ocp +++ b/build.ocp @@ -1,9 +1,20 @@ sort = true +has_byte = false +has_asm = true cflags = "-w +A" +begin library "process" + requires = [ "unix" ] + files = [ + "src/process/process.mli" + "src/process/process.ml" + ] +end + begin program "riak-snaps" - has_byte = false - has_asm = true + requires = [ + "process" + ] files = [ "src/riak_snaps/riak_snaps_main.mli" "src/riak_snaps/riak_snaps_main.ml" diff --git a/src/process/process.ml b/src/process/process.ml new file mode 100644 index 0000000..cf1a20e --- /dev/null +++ b/src/process/process.ml @@ -0,0 +1,62 @@ +module String = StringLabels +module Unix = UnixLabels + +type process_error = + | Fail of int * string + | Signal of int + | Stop of int + +type argument_error = + | Invalid_prog + +type ('ok, 'error) result = + [ `Ok of 'ok | `Error of 'error ] + +type t = + { prog : string + ; args : string list + ; stdout : in_channel + ; stdin : out_channel + ; stderr : in_channel + } + +let read_ic ~ic = + let buffer = Buffer.create 32 in + let rec read () = + try + Buffer.add_channel buffer ic 1; + read () + with End_of_file -> + () + in + read (); + Buffer.contents buffer + +let string_find ~str ~chr = + try Some (String.index str chr) with Not_found -> None + +let wait {stdout; stdin; stderr; _} = + let stdout_content = read_ic ~ic:stdout in + let stderr_content = read_ic ~ic:stderr in + match Unix.close_process_full (stdout, stdin, stderr) with + | Unix.WEXITED 0 -> `Ok stdout_content + | Unix.WEXITED n -> `Error (Fail (n, stderr_content)) + | Unix.WSIGNALED n -> `Error (Signal n) + | Unix.WSTOPPED n -> `Error (Stop n) + +let create ~prog ~args = + match string_find ~str:prog ~chr:' ' with + | Some _ -> `Error Invalid_prog + | None -> + let cmd = String.concat (prog :: args) ~sep:" " in + let env = Unix.environment () in + let stdout, stdin, stderr = Unix.open_process_full cmd ~env in + let t = + { prog + ; args + ; stdout + ; stdin + ; stderr + } + in + `Ok t diff --git a/src/process/process.mli b/src/process/process.mli new file mode 100644 index 0000000..c9b16fd --- /dev/null +++ b/src/process/process.mli @@ -0,0 +1,16 @@ +type process_error = + | Fail of int * string + | Signal of int + | Stop of int + +type argument_error = + | Invalid_prog + +type ('ok, 'error) result = + [ `Ok of 'ok | `Error of 'error ] + +type t + +val create : prog:string -> args:string list -> (t, argument_error) result + +val wait : t -> (string, process_error) result From c7fa89f34802de2b9cdc0bbaf477abcf129f894c Mon Sep 17 00:00:00 2001 From: Siraaj Khandkar Date: Sat, 22 Feb 2014 18:19:38 -0500 Subject: [PATCH 3/4] Re-write in OCaml. --- build.ocp | 2 + src/riak_snaps/riak_snaps_main.ml | 66 ++++++++++++++++++++++++++++++- 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/build.ocp b/build.ocp index f7c5897..4a63c16 100644 --- a/build.ocp +++ b/build.ocp @@ -14,6 +14,8 @@ end begin program "riak-snaps" requires = [ "process" + "str" + "ezjsonm" ] files = [ "src/riak_snaps/riak_snaps_main.mli" diff --git a/src/riak_snaps/riak_snaps_main.ml b/src/riak_snaps/riak_snaps_main.ml index e569e05..6c45f68 100644 --- a/src/riak_snaps/riak_snaps_main.ml +++ b/src/riak_snaps/riak_snaps_main.ml @@ -1,2 +1,66 @@ +open Printf + +module String = StringLabels + +let port = 8098 + +let (|-) f g x = g (f x) + +let last_line str = + List.hd (List.rev (Str.split (Str.regexp "\n+") str)) + +let sys_out ~prog ~args = + match Process.create ~prog ~args with + | `Error Process.Invalid_prog -> assert false + | `Ok proc -> + begin match Process.wait proc with + | `Ok out -> out + | `Error (Process.Signal _) -> assert false + | `Error (Process.Stop _) -> assert false + | `Error (Process.Fail (code, reason)) -> + eprintf "~~~ FAILURE ~~~\n%!"; + eprintf "Program : %s\n%!" prog; + eprintf "Arguments : %s\n%!" (String.concat args ~sep:" "); + eprintf "Exit code : %d\n%!" code; + eprintf "Reason : %s\n%!" reason; + exit code + end + +let sys_do ~prog ~args = + ignore (sys_out ~prog ~args) + +let riak_get_keys ~hostname ~bucket = + let uri = sprintf "http://%s:%d/riak/%s?keys=true" hostname port bucket in + let data = sys_out ~prog:"curl" ~args:["-i"; uri] in + let body = last_line data in + let json = Ezjsonm.from_string body in + Ezjsonm.(get_list get_string (find json ["keys"])) + +let riak_get_value ~hostname ~bucket key = + let uri = sprintf "http://%s:%d/riak/%s/%s" hostname port bucket key in + let data = sys_out ~prog:"curl" ~args:["-i"; uri] in + key, (last_line data) + +let git_init () = + sys_do ~prog:"git" ~args:["init"] + +let mkdir path = + sys_do ~prog:"mkdir" ~args:["-p"; path] + +let object_store ~bucket (key, value) = + let oc = open_out (bucket ^ "/" ^ key) in + output_string oc value; + close_out oc; + sys_do ~prog:"git" ~args:["add" ; "."]; + sys_do ~prog:"git" ~args:["commit"; "-m"; sprintf "'Update %s'" key] + let () = - print_endline "Hi" + let repo_path = Sys.argv.(1) in + let hostname = Sys.argv.(2) in + let bucket = Sys.argv.(3) in + mkdir (repo_path ^ "/" ^ bucket); + Sys.chdir repo_path; + git_init (); + List.iter + (riak_get_value ~hostname ~bucket |- object_store ~bucket) + (riak_get_keys ~hostname ~bucket) From 7add732148589ad863b6f8d886528467e71c6685 Mon Sep 17 00:00:00 2001 From: Siraaj Khandkar Date: Sat, 22 Feb 2014 18:21:40 -0500 Subject: [PATCH 4/4] Remove 1st prototype. --- riak-snapshot | 34 ---------------------------------- 1 file changed, 34 deletions(-) delete mode 100755 riak-snapshot diff --git a/riak-snapshot b/riak-snapshot deleted file mode 100755 index 132b036..0000000 --- a/riak-snapshot +++ /dev/null @@ -1,34 +0,0 @@ -#! /bin/bash - -keys2lines() { - python -c "exec(\" \ - \nimport json as j \ - \nimport sys as s \ - \nfor key in j.loads(s.stdin.read())['keys']: \ - \n print key \ - \")" -} - -main() { - repo_path=$1 - hostname=$2 - bucket=$3 - port=8098 - mkdir -p $repo_path/$bucket - cd $repo_path - git init - keys=$( - curl -i http://$hostname:$port/riak/$bucket\?keys\=true 2> /dev/null \ - | tail -1 \ - | keys2lines - ) - for key in $keys; - do - curl -i http://$hostname:$port/riak/$bucket/$key 2> /dev/null \ - | tail -1 \ - > $bucket/$key - git add . && git commit -m "Update $bucket/$key" - done -} - -main $@