merge

Makefile

@@ -12,14 +12,18 @@ bindir = $(exec_prefix)/bin
 
 all: clean test
 
-test:
+test: stage-test
 	$(SWIPL) -q -t test -l prolog/test/test
 
-test-%:
+test-%: stage-test
 	$(SWIPL) -q -t "test($*)" -l prolog/test/test
 
+stage-test:
+	(test -e t/target && rm -rf t/target) || true
+	cp -r t/init t/target
+
 clean:
-	git clean -fd t/target || true
+	git clean -fd || true
 
 symlink:
 	ln -sf $(MAKEFILE_DIR)bin/biomake $(bindir)/biomake

README.md

@@ -96,12 +96,14 @@ Options
     Assign Makefile variables from command line
 Var=Val 
     Alternative syntax for '-D Var Val'
--l DIRECTORY
-    Iterates through directory writing metadata on each file found
 -s,--quiet,--silent 
     Silent operation; do not print recipes as they are executed
 --one-shell 
     Run recipes in single shell (loosely equivalent to GNU Make's .ONESHELL)
+-y,--sync,--sync-dir URI
+    Synchronize current working directory to a remote URI. If no --sync-exec is specified, S3-form URIs (s3://mybucket/my/path) are handled using the AWS CLI tool; other URIs will be passed to rsync.
+-x,--sync-exec COMMAND
+    Specify executable for --sync.
 -H,--md5-hash 
     Use MD5 hashes instead of timestamps
 -C,--no-md5-cache 

bin/biomake

@@ -21,4 +21,4 @@ if [ -z "$BIOMAKE_PATH" ]; then
     BIOMAKE_PATH=$PATH_TO_ME/../prolog;
 fi
 
-$PATH_TO_ME/swipl_wrap -L0 -G0 -T0 -q -p library=$BIOMAKE_PATH  -g 'assert(biomake_prog("'$0'")),main,halt' -t halt -s $BIOMAKE_PATH/biomake/cli -- "$@"
+$PATH_TO_ME/swipl_wrap -q -p library=$BIOMAKE_PATH  -g 'assert(biomake_prog("'$0'")),main,halt' -t halt -s $BIOMAKE_PATH/biomake/cli -- "$@"

bin/biomake-swipl7

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Symlink resolution: http://stackoverflow.com/a/697552/726581
+
+# get the absolute path of the executable
+SELF_PATH=$(cd -P -- "$(dirname -- "$0")" && pwd -P) && SELF_PATH=$SELF_PATH/$(basename -- "$0")
+
+# resolve symlinks
+while [[ -h $SELF_PATH ]]; do
+    # 1) cd to directory of the symlink
+    # 2) cd to the directory of where the symlink points
+    # 3) get the pwd
+    # 4) append the basename
+    DIR=$(dirname -- "$SELF_PATH")
+    SYM=$(readlink "$SELF_PATH")
+    SELF_PATH=$(cd "$DIR" && cd "$(dirname -- "$SYM")" && pwd)/$(basename -- "$SYM")
+done
+
+PATH_TO_ME=`dirname $SELF_PATH`;
+if [ -z "$BIOMAKE_PATH" ]; then
+    BIOMAKE_PATH=$PATH_TO_ME/../prolog;
+fi
+
+$PATH_TO_ME/swipl_wrap -L0 -G0 -T0 -q -p library=$BIOMAKE_PATH  -g 'assert(biomake_prog("'$0'")),main,halt' -t halt -s $BIOMAKE_PATH/biomake/cli -- "$@"

prolog/biomake/biomake.pl

@@ -470,11 +470,11 @@
 	rule_target(Rule,T,Opts),
 	(get_opt(md5,true,Opts) -> ensure_md5_directory_exists(T) ; true),
 	run_execs_in_queue(Q,Rule,SL,Opts),
-	verbose_report('~w queued for rebuild',[T],SL,Opts).
+	verbose_report('Target ~w queued for rebuild',[T],SL,Opts).
 dispatch_run_execs(Rule,SL,Opts) :-
 	run_execs_now(Rule,SL,Opts),
 	rule_target(Rule,T,Opts),
-	verbose_report('~w built',[T],SL,Opts).
+	verbose_report('Target ~w built',[T],SL,Opts).
 
 run_execs_now(Rule,SL,Opts) :-
 	get_opt(oneshell,true,Opts),

prolog/biomake/cli.pl

@@ -3,6 +3,7 @@
 :- use_module(library(biomake/biomake)).
 :- use_module(library(biomake/utils)).
 :- use_module(library(biomake/embed)).
+:- use_module(library(biomake/sync)).
 
 % ----------------------------------------
 % MAIN PROGRAM
@@ -13,7 +14,6 @@
 :- nodebug(build).
 :- nodebug(md5).
 
-
 main :-
         current_prolog_flag(argv, Arguments),
         (append(_SytemArgs, [--|Args], Arguments) ; =(Arguments,Args)),
@@ -30,9 +30,11 @@
                G),
         forall(member(flush_queue(T),AllOpts),
 	       flush_queue_recursive(T,AllOpts)),
+	sync_remote_to_cwd(Cwd,AllOpts),
 	(build_toplevel(AllOpts)
-	 -> halt_success
-	 ;  halt_error).
+	-> sync_dir_to_remote(Cwd,AllOpts),
+	   halt_success
+	;  halt_error).
 
 build_toplevel(Opts) :-
 	member(toplevel(_),Opts),
@@ -294,14 +296,6 @@
 % ESOTERIC FEATURES
 % ----------------------------------------
 
-parse_arg(['-l',F|L],L,
-          goal( (collect_stored_targets(F,[]),
-                 show_stored_targets
-                ) )) :-
-        ensure_loaded(library(biomake/scan)),
-        !.
-arg_info('-l','DIRECTORY','Iterates through directory writing metadata on each file found').
-
 simple_arg('-s',silent(true)).
 arg_alias('-s','--quiet').
 arg_alias('-s','--silent').
@@ -310,6 +304,26 @@
 simple_arg('--one-shell',oneshell(true)).
 arg_info('--one-shell','','Run recipes in single shell (loosely equivalent to GNU Make\'s .ONESHELL)').
 
+% ----------------------------------------
+% SYNC TO REMOTE STORAGE
+% ----------------------------------------
+
+parse_arg(['-y',URIs|L],L,sync(URI)) :-
+	atom_string(URI,URIs),
+	!.
+arg_alias('-y','--sync').
+arg_alias('-y','--sync-dir').
+recover_arg(['-y',URI],sync(URI)).
+arg_info('-y','URI','Synchronize current working directory to a remote URI. If no --sync-exec is specified, S3-form URIs (s3://mybucket/my/path) are handled using the AWS CLI tool; other URIs will be passed to rsync.').
+
+parse_arg(['-x',Es|L],L,sync_exec(E)) :-
+	atom_string(E,Es),
+	!.
+arg_alias('-x','--sync-exec').
+recover_arg(['-x',E],sync_exec(E)).
+arg_info('-x','COMMAND','Specify executable for --sync.').
+
+
 % ----------------------------------------
 % MD5 CHECKSUMS
 % ----------------------------------------

prolog/biomake/sync.pl

@@ -0,0 +1,62 @@
+% * -*- Mode: Prolog -*- */
+
+:- module(sync,
+          [
+	      sync_remote_to_cwd/2,
+	      sync_remote_to_dir/2,
+	      sync_dir_to_remote/2
+          ]).
+
+:- use_module(library(biomake/biomake)).
+
+% ----------------------------------------
+% SYNC TO/FROM REMOTE STORAGE
+% ----------------------------------------
+
+sync_remote_to_cwd(Dir,Opts) :-
+    working_directory(Cwd,Cwd),
+    absolute_file_name(Cwd,Dir),
+    sync_remote_to_dir(Dir,Opts).
+
+sync_remote_to_dir(Dir,Opts) :-
+    get_opt(sync,URI,Opts),
+    !,
+    sync_exec(Exec,URI,Opts),
+    ends_with_slash(URI,URISlash),
+    ends_with_slash(Dir,DirSlash),
+    verbose_report('Syncing from ~w to ~w',[URISlash,DirSlash],Opts),
+    format(string(SyncCmd),"~w ~w ~w",[Exec,URISlash,DirSlash]),
+    shell(SyncCmd).
+
+sync_remote_to_dir(_Dir,_Opts).
+
+sync_dir_to_remote(Dir,Opts) :-
+    get_opt(sync,URI,Opts),
+    !,
+    sync_exec(Exec,URI,Opts),
+    ends_with_slash(URI,URISlash),
+    ends_with_slash(Dir,DirSlash),
+    verbose_report('Syncing from ~w to ~w',[DirSlash,URISlash],Opts),
+    format(string(SyncCmd),"~w ~w ~w",[Exec,DirSlash,URISlash]),
+    shell(SyncCmd).
+
+sync_dir_to_remote(_Dir,_Opts).
+
+ends_with_slash(S,S) :- string_concat(_,"/",S), !.
+ends_with_slash(S,Ss) :- string_concat(S,"/",Ss).
+
+sync_exec(Exec,_URI,Opts) :-
+    get_opt(sync_exec,Exec,Opts),
+    !.
+
+sync_exec(Exec,URI,_Opts) :-
+    string_concat("s3://",_,URI),
+    s3_sync_exec(Exec),
+    !.
+
+sync_exec(Exec,_URI,_Opts) :-
+    rsync_exec(Exec).
+
+s3_sync_exec("aws s3 sync --delete").
+
+rsync_exec("rsync -r --delete").

prolog/test/test.pl

@@ -316,6 +316,9 @@
 	run_test("-f Makefile.regex","testregex_apple"),
 	run_failure_test("-f Makefile.regex","testregex_cat"),
 	run_failure_test("-f Makefile.regex","testregex_ALBACORE"),  % fails due to default case-sensitivity of regexes
+
+	announce("REMOTE SYNC"),
+	run_test("ref/sync","target/sync",[],[],"","all"),
 
 	% All done
 	report_counts,

t/init/sync/Makefile

@@ -0,0 +1,6 @@
+
+all:
+	cd local; $(MAKE) -f ../Makefile --sync ../remote C.bar
+
+%.bar: %.foo
+	perl -pe 'tr/a-z/A-Z/' $< > $@

t/init/sync/local/A.txt

@@ -0,0 +1 @@
+Apples

t/init/sync/local/B.txt

@@ -0,0 +1 @@
+Bears

t/init/sync/local/C.foo

@@ -0,0 +1 @@
+Clauses

t/init/sync/remote/A.txt

@@ -0,0 +1 @@
+Apples

t/init/sync/remote/B.txt

@@ -0,0 +1 @@
+Bears

t/init/sync/remote/C.foo

@@ -0,0 +1 @@
+Clauses

t/ref/sync/Makefile

@@ -0,0 +1,6 @@
+
+all:
+	cd local; $(MAKE) -f ../Makefile --sync ../remote C.bar
+
+%.bar: %.foo
+	perl -pe 'tr/a-z/A-Z/' $< > $@

t/ref/sync/local/A.txt

@@ -0,0 +1 @@
+Apples

t/ref/sync/local/B.txt

@@ -0,0 +1 @@
+Bears

t/ref/sync/local/C.bar

@@ -0,0 +1 @@
+CLAUSES

t/ref/sync/local/C.foo

@@ -0,0 +1 @@
+Clauses

t/ref/sync/remote/A.txt

@@ -0,0 +1 @@
+Apples

t/ref/sync/remote/B.txt

@@ -0,0 +1 @@
+Bears

t/ref/sync/remote/C.bar

@@ -0,0 +1 @@
+CLAUSES

t/ref/sync/remote/C.foo

@@ -0,0 +1 @@
+Clauses