From 1dd436adc7fbc070916a10be68a67f7c05075682 Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Tue, 1 Apr 2014 16:41:01 -0700 Subject: [PATCH] Initial support for new hostbin startup mode --- config.h.in | 3 + configure | 53 +- configure.common.ac | 4 + src/client/beboot/spindle_bootstrap.c | 71 ++- src/client/config.h.in | 3 + src/client/configure | 41 +- src/fe/Makefile.am | 3 +- src/fe/Makefile.in | 5 +- src/fe/config.h.in | 3 + src/fe/configure | 56 +- src/fe/configure.ac | 2 +- src/fe/hostbin/Makefile.am | 4 + src/fe/hostbin/Makefile.in | 514 +++++++++++++++++++ src/fe/hostbin/launch_hostbin.cc | 595 ++++++++++++++++++++++ src/fe/launchmon/spindle_fe_lmon.cc | 1 - src/fe/openmpi_intercept/ompi_intercept.c | 2 +- src/fe/openmpi_intercept/parse_openmpi.cc | 6 +- src/fe/startup/Makefile.am | 2 +- src/fe/startup/Makefile.in | 9 +- src/fe/startup/parse_launcher.cc | 19 +- src/fe/startup/parse_launcher.h | 3 + src/fe/startup/parseargs.cc | 33 ++ src/fe/startup/parseargs.h | 2 + src/fe/startup/spindle_fe.cc | 3 +- src/fe/startup/spindle_fe_main.cc | 63 ++- src/include/spindle_launch.h | 8 + src/server/config.h.in | 3 + src/server/configure | 53 +- src/server/startup/Makefile.am | 2 +- src/server/startup/Makefile.in | 20 +- src/server/startup/spindle_be.cc | 1 + src/server/startup/spindle_be_hostbin.cc | 23 + src/server/startup/spindle_be_main.cc | 27 +- 33 files changed, 1512 insertions(+), 125 deletions(-) create mode 100644 src/fe/hostbin/Makefile.am create mode 100644 src/fe/hostbin/Makefile.in create mode 100644 src/fe/hostbin/launch_hostbin.cc create mode 100644 src/server/startup/spindle_be_hostbin.cc diff --git a/config.h.in b/config.h.in index f93344e3..57c9bd34 100644 --- a/config.h.in +++ b/config.h.in @@ -51,6 +51,9 @@ /* Define to 1 if you have the header file. */ #undef HAVE_UNISTD_H +/* Hostbin executable */ +#undef HOSTBIN_PATH + /* Number of bytes in keys for gcrypt */ #undef KEY_SIZE_BYTES diff --git a/configure b/configure index 3819de11..aa133eff 100755 --- a/configure +++ b/configure @@ -969,6 +969,7 @@ with_testrm with_default_port with_localstorage enable_bluegene +with_hostbin enable_pipes enable_socket enable_shmem @@ -1679,6 +1680,7 @@ Optional Packages: 'slurm' and 'flux') --with-default-port=NUM TCP Port for Spindle server communication --with-localstorage=DIR Directory on back-ends for storing relocated files + --with-hostbin=EXE Executable for returning host lists for jobs --with-python-prefix=STR List of directories containing python installs --with-usage-logging=FILE @@ -5131,13 +5133,13 @@ if test "${lt_cv_nm_interface+set}" = set; then else lt_cv_nm_interface="BSD nm" echo "int some_variable = 0;" > conftest.$ac_ext - (eval echo "\"\$as_me:5134: $ac_compile\"" >&5) + (eval echo "\"\$as_me:5136: $ac_compile\"" >&5) (eval "$ac_compile" 2>conftest.err) cat conftest.err >&5 - (eval echo "\"\$as_me:5137: $NM \\\"conftest.$ac_objext\\\"\"" >&5) + (eval echo "\"\$as_me:5139: $NM \\\"conftest.$ac_objext\\\"\"" >&5) (eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out) cat conftest.err >&5 - (eval echo "\"\$as_me:5140: output\"" >&5) + (eval echo "\"\$as_me:5142: output\"" >&5) cat conftest.out >&5 if $GREP 'External.*some_variable' conftest.out > /dev/null; then lt_cv_nm_interface="MS dumpbin" @@ -6343,7 +6345,7 @@ ia64-*-hpux*) ;; *-*-irix6*) # Find out which ABI we are using. - echo '#line 6346 "configure"' > conftest.$ac_ext + echo '#line 6348 "configure"' > conftest.$ac_ext if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5 (eval $ac_compile) 2>&5 ac_status=$? @@ -8934,11 +8936,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:8937: $lt_compile\"" >&5) + (eval echo "\"\$as_me:8939: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:8941: \$? = $ac_status" >&5 + echo "$as_me:8943: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. @@ -9273,11 +9275,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:9276: $lt_compile\"" >&5) + (eval echo "\"\$as_me:9278: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:9280: \$? = $ac_status" >&5 + echo "$as_me:9282: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. @@ -9378,11 +9380,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:9381: $lt_compile\"" >&5) + (eval echo "\"\$as_me:9383: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:9385: \$? = $ac_status" >&5 + echo "$as_me:9387: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized @@ -9433,11 +9435,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:9436: $lt_compile\"" >&5) + (eval echo "\"\$as_me:9438: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:9440: \$? = $ac_status" >&5 + echo "$as_me:9442: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized @@ -12236,7 +12238,7 @@ else lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2 lt_status=$lt_dlunknown cat > conftest.$ac_ext <<_LT_EOF -#line 12239 "configure" +#line 12241 "configure" #include "confdefs.h" #if HAVE_DLFCN_H @@ -12332,7 +12334,7 @@ else lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2 lt_status=$lt_dlunknown cat > conftest.$ac_ext <<_LT_EOF -#line 12335 "configure" +#line 12337 "configure" #include "confdefs.h" #if HAVE_DLFCN_H @@ -14352,11 +14354,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:14355: $lt_compile\"" >&5) + (eval echo "\"\$as_me:14357: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:14359: \$? = $ac_status" >&5 + echo "$as_me:14361: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. @@ -14451,11 +14453,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:14454: $lt_compile\"" >&5) + (eval echo "\"\$as_me:14456: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:14458: \$? = $ac_status" >&5 + echo "$as_me:14460: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized @@ -14503,11 +14505,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:14506: $lt_compile\"" >&5) + (eval echo "\"\$as_me:14508: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:14510: \$? = $ac_status" >&5 + echo "$as_me:14512: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized @@ -16076,6 +16078,17 @@ if test "x$OS_BUILD" == "x"; then OS_BUILD=linux fi +#Startup type + +# Check whether --with-hostbin was given. +if test "${with_hostbin+set}" = set; then + withval=$with_hostbin; +cat >>confdefs.h <<_ACEOF +#define HOSTBIN_PATH "${withval}" +_ACEOF + +fi + #Runmode detection (pipe/socket or cobo/msocket communications) if test "x$OS_BUILD" == "xlinux"; then diff --git a/configure.common.ac b/configure.common.ac index 72ac0a51..60c07b18 100644 --- a/configure.common.ac +++ b/configure.common.ac @@ -21,6 +21,10 @@ if test "x$OS_BUILD" == "x"; then OS_BUILD=linux fi +#Startup type +AC_ARG_WITH(hostbin, + [AS_HELP_STRING([--with-hostbin=EXE],[Executable for returning host lists for jobs])], + [AC_DEFINE_UNQUOTED([HOSTBIN_PATH],"[${withval}]",[Hostbin executable])],) #Runmode detection (pipe/socket or cobo/msocket communications) if test "x$OS_BUILD" == "xlinux"; then diff --git a/src/client/beboot/spindle_bootstrap.c b/src/client/beboot/spindle_bootstrap.c index ce9c21df..9afdfe00 100644 --- a/src/client/beboot/spindle_bootstrap.c +++ b/src/client/beboot/spindle_bootstrap.c @@ -21,6 +21,8 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include #include #include +#include +#include #include #include @@ -36,7 +38,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA #if !defined(LIBEXECDIR) #error Expected to be built with libdir defined #endif - +char spindle_daemon[] = LIBEXECDIR "/spindle_be"; int ldcsid; static int rankinfo[4]={-1,-1,-1,-1}; @@ -46,6 +48,7 @@ static char **cmdline; static char *executable; static char *client_lib; static char *opts_s; +static char **daemon_args; unsigned long opts; @@ -98,19 +101,71 @@ static void setup_environment() static int parse_cmdline(int argc, char *argv[]) { + int i, daemon_arg_count; if (argc < 4) return -1; + + i = 1; + if (strcmp(argv[1], "-daemon_args") == 0) { + debug_printf("Parsing daemon args out of bootstrap launch line\n"); + daemon_arg_count = atoi(argv[2]); + daemon_args = malloc(sizeof(char *) * (daemon_arg_count + 2)); + daemon_args[0] = spindle_daemon; + for (i = 4; i < 3 + daemon_arg_count; i++) + daemon_args[i - 3] = argv[i]; + daemon_args[i - 3] = NULL; + } - location = argv[1]; - number_s = argv[2]; + location = argv[i++]; + number_s = argv[i++]; number = atoi(number_s); - opts_s = argv[3]; + opts_s = argv[i++]; opts = atoi(opts_s); - cmdline = argv + 4; + cmdline = argv + i; + assert(i < argc); return 0; } +static void launch_daemon(char *location) +{ + /*grand-child fork, then execv daemon. By grand-child forking we ensure that + the app won't get confused by seeing an unknown process as a child. */ + pid_t child, gchild; + int status; + int fd; + char unique_file[MAX_PATH_LEN+1]; + char buffer[32]; + + snprintf(unique_file, MAX_PATH_LEN, "%s/spindle_daemon_pid", location); + unique_file[MAX_PATH_LEN] = '\0'; + fd = open(unique_file, O_CREAT | O_EXCL | O_WRONLY, 0600); + if (fd == -1) { + debug_printf("Not starting daemon -- %s already exists\n", unique_file); + return; + } + debug_printf("Client is spawning daemon\n"); + + child = fork(); + if (child == 0) { + gchild = fork(); + if (gchild != 0) { + snprintf(buffer, sizeof(buffer), "%d\n", getpid()); + write(fd, buffer, strlen(buffer)); + close(fd); + exit(0); + } + close(fd); + execv(spindle_daemon, daemon_args); + fprintf(stderr, "Spindle error: Could not execv daemon %s\n", daemon_args[0]); + exit(-1); + } + else if (child > 0) { + close(fd); + waitpid(child, &status, 0); + } +} + static void get_executable() { if (!(opts & OPT_RELOCAOUT)) { @@ -129,8 +184,6 @@ static void get_executable() debug_printf("Relocated executable %s to %s\n", *cmdline, executable); chmod(executable, 0700); } - - } static void adjust_script() @@ -226,6 +279,10 @@ int main(int argc, char *argv[]) return -1; } location = realize(location); + + if (daemon_args) { + launch_daemon(location); + } result = establish_connection(); if (result == -1) { diff --git a/src/client/config.h.in b/src/client/config.h.in index a3702239..b160ceaf 100644 --- a/src/client/config.h.in +++ b/src/client/config.h.in @@ -45,6 +45,9 @@ /* Define to 1 if you have the header file. */ #undef HAVE_UNISTD_H +/* Hostbin executable */ +#undef HOSTBIN_PATH + /* Number of bytes in keys for gcrypt */ #undef KEY_SIZE_BYTES diff --git a/src/client/configure b/src/client/configure index a74517b3..9166909a 100755 --- a/src/client/configure +++ b/src/client/configure @@ -933,6 +933,7 @@ enable_libtool_lock with_default_port with_localstorage enable_bluegene +with_hostbin enable_pipes enable_socket enable_shmem @@ -1619,6 +1620,7 @@ Optional Packages: --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-default-port=NUM TCP Port for Spindle server communication --with-localstorage=DIR Directory on back-ends for storing relocated files + --with-hostbin=EXE Executable for returning host lists for jobs --with-python-prefix=STR List of directories containing python installs --with-usage-logging=FILE @@ -4488,13 +4490,13 @@ if test "${lt_cv_nm_interface+set}" = set; then else lt_cv_nm_interface="BSD nm" echo "int some_variable = 0;" > conftest.$ac_ext - (eval echo "\"\$as_me:4491: $ac_compile\"" >&5) + (eval echo "\"\$as_me:4493: $ac_compile\"" >&5) (eval "$ac_compile" 2>conftest.err) cat conftest.err >&5 - (eval echo "\"\$as_me:4494: $NM \\\"conftest.$ac_objext\\\"\"" >&5) + (eval echo "\"\$as_me:4496: $NM \\\"conftest.$ac_objext\\\"\"" >&5) (eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out) cat conftest.err >&5 - (eval echo "\"\$as_me:4497: output\"" >&5) + (eval echo "\"\$as_me:4499: output\"" >&5) cat conftest.out >&5 if $GREP 'External.*some_variable' conftest.out > /dev/null; then lt_cv_nm_interface="MS dumpbin" @@ -5700,7 +5702,7 @@ ia64-*-hpux*) ;; *-*-irix6*) # Find out which ABI we are using. - echo '#line 5703 "configure"' > conftest.$ac_ext + echo '#line 5705 "configure"' > conftest.$ac_ext if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5 (eval $ac_compile) 2>&5 ac_status=$? @@ -7553,11 +7555,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:7556: $lt_compile\"" >&5) + (eval echo "\"\$as_me:7558: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:7560: \$? = $ac_status" >&5 + echo "$as_me:7562: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. @@ -7892,11 +7894,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:7895: $lt_compile\"" >&5) + (eval echo "\"\$as_me:7897: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:7899: \$? = $ac_status" >&5 + echo "$as_me:7901: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. @@ -7997,11 +7999,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:8000: $lt_compile\"" >&5) + (eval echo "\"\$as_me:8002: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:8004: \$? = $ac_status" >&5 + echo "$as_me:8006: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized @@ -8052,11 +8054,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:8055: $lt_compile\"" >&5) + (eval echo "\"\$as_me:8057: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:8059: \$? = $ac_status" >&5 + echo "$as_me:8061: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized @@ -10855,7 +10857,7 @@ else lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2 lt_status=$lt_dlunknown cat > conftest.$ac_ext <<_LT_EOF -#line 10858 "configure" +#line 10860 "configure" #include "confdefs.h" #if HAVE_DLFCN_H @@ -10951,7 +10953,7 @@ else lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2 lt_status=$lt_dlunknown cat > conftest.$ac_ext <<_LT_EOF -#line 10954 "configure" +#line 10956 "configure" #include "confdefs.h" #if HAVE_DLFCN_H @@ -11265,6 +11267,17 @@ if test "x$OS_BUILD" == "x"; then OS_BUILD=linux fi +#Startup type + +# Check whether --with-hostbin was given. +if test "${with_hostbin+set}" = set; then + withval=$with_hostbin; +cat >>confdefs.h <<_ACEOF +#define HOSTBIN_PATH "${withval}" +_ACEOF + +fi + #Runmode detection (pipe/socket or cobo/msocket communications) if test "x$OS_BUILD" == "xlinux"; then diff --git a/src/fe/Makefile.am b/src/fe/Makefile.am index 6994fa91..bdc9bfcd 100644 --- a/src/fe/Makefile.am +++ b/src/fe/Makefile.am @@ -2,5 +2,6 @@ if LMON BUILD_LMON=launchmon endif -SUBDIRS = logging cobo comlib openmpi_intercept $(BUILD_LMON) startup +SUBDIRS = logging cobo comlib openmpi_intercept $(BUILD_LMON) hostbin startup + diff --git a/src/fe/Makefile.in b/src/fe/Makefile.in index 63365266..65a9eb91 100644 --- a/src/fe/Makefile.in +++ b/src/fe/Makefile.in @@ -81,7 +81,8 @@ AM_RECURSIVE_TARGETS = $(RECURSIVE_TARGETS:-recursive=) \ distdir dist dist-all distcheck ETAGS = etags CTAGS = ctags -DIST_SUBDIRS = logging cobo comlib openmpi_intercept launchmon startup +DIST_SUBDIRS = logging cobo comlib openmpi_intercept launchmon hostbin \ + startup DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) distdir = $(PACKAGE)-$(VERSION) top_distdir = $(distdir) @@ -249,7 +250,7 @@ top_build_prefix = @top_build_prefix@ top_builddir = @top_builddir@ top_srcdir = @top_srcdir@ @LMON_TRUE@BUILD_LMON = launchmon -SUBDIRS = logging cobo comlib openmpi_intercept $(BUILD_LMON) startup +SUBDIRS = logging cobo comlib openmpi_intercept $(BUILD_LMON) hostbin startup all: config.h $(MAKE) $(AM_MAKEFLAGS) all-recursive diff --git a/src/fe/config.h.in b/src/fe/config.h.in index aa84257c..c84b6a16 100644 --- a/src/fe/config.h.in +++ b/src/fe/config.h.in @@ -69,6 +69,9 @@ /* Define to 1 if you have the header file. */ #undef HAVE_UNISTD_H +/* Hostbin executable */ +#undef HOSTBIN_PATH + /* Use keyfile for authentication */ #undef KEYFILE diff --git a/src/fe/configure b/src/fe/configure index 480c6bdc..29811c35 100755 --- a/src/fe/configure +++ b/src/fe/configure @@ -954,6 +954,7 @@ enable_libtool_lock with_default_port with_localstorage enable_bluegene +with_hostbin enable_pipes enable_socket enable_shmem @@ -1657,6 +1658,7 @@ Optional Packages: --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-default-port=NUM TCP Port for Spindle server communication --with-localstorage=DIR Directory on back-ends for storing relocated files + --with-hostbin=EXE Executable for returning host lists for jobs --with-python-prefix=STR List of directories containing python installs --with-usage-logging=FILE @@ -4419,7 +4421,7 @@ else fi -ac_config_files="$ac_config_files logging/Makefile cobo/Makefile comlib/Makefile openmpi_intercept/Makefile launchmon/Makefile startup/Makefile Makefile" +ac_config_files="$ac_config_files logging/Makefile cobo/Makefile comlib/Makefile openmpi_intercept/Makefile launchmon/Makefile startup/Makefile hostbin/Makefile Makefile" case `pwd` in *\ * | *\ *) @@ -5040,13 +5042,13 @@ if test "${lt_cv_nm_interface+set}" = set; then else lt_cv_nm_interface="BSD nm" echo "int some_variable = 0;" > conftest.$ac_ext - (eval echo "\"\$as_me:5043: $ac_compile\"" >&5) + (eval echo "\"\$as_me:5045: $ac_compile\"" >&5) (eval "$ac_compile" 2>conftest.err) cat conftest.err >&5 - (eval echo "\"\$as_me:5046: $NM \\\"conftest.$ac_objext\\\"\"" >&5) + (eval echo "\"\$as_me:5048: $NM \\\"conftest.$ac_objext\\\"\"" >&5) (eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out) cat conftest.err >&5 - (eval echo "\"\$as_me:5049: output\"" >&5) + (eval echo "\"\$as_me:5051: output\"" >&5) cat conftest.out >&5 if $GREP 'External.*some_variable' conftest.out > /dev/null; then lt_cv_nm_interface="MS dumpbin" @@ -6252,7 +6254,7 @@ ia64-*-hpux*) ;; *-*-irix6*) # Find out which ABI we are using. - echo '#line 6255 "configure"' > conftest.$ac_ext + echo '#line 6257 "configure"' > conftest.$ac_ext if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5 (eval $ac_compile) 2>&5 ac_status=$? @@ -8843,11 +8845,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:8846: $lt_compile\"" >&5) + (eval echo "\"\$as_me:8848: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:8850: \$? = $ac_status" >&5 + echo "$as_me:8852: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. @@ -9182,11 +9184,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:9185: $lt_compile\"" >&5) + (eval echo "\"\$as_me:9187: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:9189: \$? = $ac_status" >&5 + echo "$as_me:9191: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. @@ -9287,11 +9289,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:9290: $lt_compile\"" >&5) + (eval echo "\"\$as_me:9292: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:9294: \$? = $ac_status" >&5 + echo "$as_me:9296: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized @@ -9342,11 +9344,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:9345: $lt_compile\"" >&5) + (eval echo "\"\$as_me:9347: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:9349: \$? = $ac_status" >&5 + echo "$as_me:9351: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized @@ -12145,7 +12147,7 @@ else lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2 lt_status=$lt_dlunknown cat > conftest.$ac_ext <<_LT_EOF -#line 12148 "configure" +#line 12150 "configure" #include "confdefs.h" #if HAVE_DLFCN_H @@ -12241,7 +12243,7 @@ else lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2 lt_status=$lt_dlunknown cat > conftest.$ac_ext <<_LT_EOF -#line 12244 "configure" +#line 12246 "configure" #include "confdefs.h" #if HAVE_DLFCN_H @@ -14261,11 +14263,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:14264: $lt_compile\"" >&5) + (eval echo "\"\$as_me:14266: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:14268: \$? = $ac_status" >&5 + echo "$as_me:14270: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. @@ -14360,11 +14362,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:14363: $lt_compile\"" >&5) + (eval echo "\"\$as_me:14365: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:14367: \$? = $ac_status" >&5 + echo "$as_me:14369: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized @@ -14412,11 +14414,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:14415: $lt_compile\"" >&5) + (eval echo "\"\$as_me:14417: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:14419: \$? = $ac_status" >&5 + echo "$as_me:14421: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized @@ -15486,6 +15488,17 @@ if test "x$OS_BUILD" == "x"; then OS_BUILD=linux fi +#Startup type + +# Check whether --with-hostbin was given. +if test "${with_hostbin+set}" = set; then + withval=$with_hostbin; +cat >>confdefs.h <<_ACEOF +#define HOSTBIN_PATH "${withval}" +_ACEOF + +fi + #Runmode detection (pipe/socket or cobo/msocket communications) if test "x$OS_BUILD" == "xlinux"; then @@ -18060,6 +18073,7 @@ do "openmpi_intercept/Makefile") CONFIG_FILES="$CONFIG_FILES openmpi_intercept/Makefile" ;; "launchmon/Makefile") CONFIG_FILES="$CONFIG_FILES launchmon/Makefile" ;; "startup/Makefile") CONFIG_FILES="$CONFIG_FILES startup/Makefile" ;; + "hostbin/Makefile") CONFIG_FILES="$CONFIG_FILES hostbin/Makefile" ;; "Makefile") CONFIG_FILES="$CONFIG_FILES Makefile" ;; "libtool") CONFIG_COMMANDS="$CONFIG_COMMANDS libtool" ;; diff --git a/src/fe/configure.ac b/src/fe/configure.ac index 9e5adda9..330a9024 100644 --- a/src/fe/configure.ac +++ b/src/fe/configure.ac @@ -7,7 +7,7 @@ AM_SILENT_RULES([yes]) AC_CONFIG_HEADERS([config.h]) AC_PROG_CC AC_PROG_CXX -AC_CONFIG_FILES([logging/Makefile cobo/Makefile comlib/Makefile openmpi_intercept/Makefile launchmon/Makefile startup/Makefile Makefile]) +AC_CONFIG_FILES([logging/Makefile cobo/Makefile comlib/Makefile openmpi_intercept/Makefile launchmon/Makefile startup/Makefile hostbin/Makefile Makefile]) LT_INIT #Include common ops diff --git a/src/fe/hostbin/Makefile.am b/src/fe/hostbin/Makefile.am new file mode 100644 index 00000000..9ca52049 --- /dev/null +++ b/src/fe/hostbin/Makefile.am @@ -0,0 +1,4 @@ +noinst_LTLIBRARIES = libhostbin.la + +libhostbin_la_SOURCES = launch_hostbin.cc +libhostbin_la_CPPFLAGS = -I$(top_srcdir)/../logging -I$(top_srcdir)/../include -DLIBEXEC=\"${pkglibexecdir}\" diff --git a/src/fe/hostbin/Makefile.in b/src/fe/hostbin/Makefile.in new file mode 100644 index 00000000..f1b06ef4 --- /dev/null +++ b/src/fe/hostbin/Makefile.in @@ -0,0 +1,514 @@ +# Makefile.in generated by automake 1.11.1 from Makefile.am. +# @configure_input@ + +# Copyright (C) 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, +# 2003, 2004, 2005, 2006, 2007, 2008, 2009 Free Software Foundation, +# Inc. +# This Makefile.in is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +@SET_MAKE@ + +VPATH = @srcdir@ +pkgdatadir = $(datadir)/@PACKAGE@ +pkgincludedir = $(includedir)/@PACKAGE@ +pkglibdir = $(libdir)/@PACKAGE@ +pkglibexecdir = $(libexecdir)/@PACKAGE@ +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd +install_sh_DATA = $(install_sh) -c -m 644 +install_sh_PROGRAM = $(install_sh) -c +install_sh_SCRIPT = $(install_sh) -c +INSTALL_HEADER = $(INSTALL_DATA) +transform = $(program_transform_name) +NORMAL_INSTALL = : +PRE_INSTALL = : +POST_INSTALL = : +NORMAL_UNINSTALL = : +PRE_UNINSTALL = : +POST_UNINSTALL = : +build_triplet = @build@ +host_triplet = @host@ +target_triplet = @target@ +subdir = hostbin +DIST_COMMON = $(srcdir)/Makefile.am $(srcdir)/Makefile.in +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 +am__aclocal_m4_deps = $(top_srcdir)/../../m4/lx_detect_bluegene.m4 \ + $(top_srcdir)/../../configure.common.ac \ + $(top_srcdir)/../../m4/lmon.m4 \ + $(top_srcdir)/../../m4/launcher.m4 $(top_srcdir)/configure.ac +am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ + $(ACLOCAL_M4) +mkinstalldirs = $(install_sh) -d +CONFIG_HEADER = $(top_builddir)/config.h +CONFIG_CLEAN_FILES = +CONFIG_CLEAN_VPATH_FILES = +LTLIBRARIES = $(noinst_LTLIBRARIES) +libhostbin_la_LIBADD = +am_libhostbin_la_OBJECTS = libhostbin_la-launch_hostbin.lo +libhostbin_la_OBJECTS = $(am_libhostbin_la_OBJECTS) +AM_V_lt = $(am__v_lt_$(V)) +am__v_lt_ = $(am__v_lt_$(AM_DEFAULT_VERBOSITY)) +am__v_lt_0 = --silent +DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir) +depcomp = $(SHELL) $(top_srcdir)/../../scripts/depcomp +am__depfiles_maybe = depfiles +am__mv = mv -f +CXXCOMPILE = $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \ + $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) +LTCXXCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) \ + $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \ + $(AM_CXXFLAGS) $(CXXFLAGS) +AM_V_CXX = $(am__v_CXX_$(V)) +am__v_CXX_ = $(am__v_CXX_$(AM_DEFAULT_VERBOSITY)) +am__v_CXX_0 = @echo " CXX " $@; +AM_V_at = $(am__v_at_$(V)) +am__v_at_ = $(am__v_at_$(AM_DEFAULT_VERBOSITY)) +am__v_at_0 = @ +CXXLD = $(CXX) +CXXLINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CXXLD) $(AM_CXXFLAGS) \ + $(CXXFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@ +AM_V_CXXLD = $(am__v_CXXLD_$(V)) +am__v_CXXLD_ = $(am__v_CXXLD_$(AM_DEFAULT_VERBOSITY)) +am__v_CXXLD_0 = @echo " CXXLD " $@; +AM_V_GEN = $(am__v_GEN_$(V)) +am__v_GEN_ = $(am__v_GEN_$(AM_DEFAULT_VERBOSITY)) +am__v_GEN_0 = @echo " GEN " $@; +SOURCES = $(libhostbin_la_SOURCES) +DIST_SOURCES = $(libhostbin_la_SOURCES) +ETAGS = etags +CTAGS = ctags +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) +ACLOCAL = @ACLOCAL@ +AMTAR = @AMTAR@ +AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ +AR = @AR@ +AUTOCONF = @AUTOCONF@ +AUTOHEADER = @AUTOHEADER@ +AUTOMAKE = @AUTOMAKE@ +AWK = @AWK@ +CC = @CC@ +CCDEPMODE = @CCDEPMODE@ +CFLAGS = @CFLAGS@ +CPP = @CPP@ +CPPFLAGS = @CPPFLAGS@ +CXX = @CXX@ +CXXCPP = @CXXCPP@ +CXXDEPMODE = @CXXDEPMODE@ +CXXFLAGS = @CXXFLAGS@ +CYGPATH_W = @CYGPATH_W@ +DEFS = @DEFS@ +DEPDIR = @DEPDIR@ +DSYMUTIL = @DSYMUTIL@ +DUMPBIN = @DUMPBIN@ +ECHO_C = @ECHO_C@ +ECHO_N = @ECHO_N@ +ECHO_T = @ECHO_T@ +EGREP = @EGREP@ +EXEEXT = @EXEEXT@ +FGREP = @FGREP@ +GCRYPT_CFLAGS = @GCRYPT_CFLAGS@ +GCRYPT_LIBS = @GCRYPT_LIBS@ +GREP = @GREP@ +HAVE_LMON = @HAVE_LMON@ +INSTALL = @INSTALL@ +INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ +LAUNCHMON_INC = @LAUNCHMON_INC@ +LAUNCHMON_LIB = @LAUNCHMON_LIB@ +LAUNCHMON_RMCOMM = @LAUNCHMON_RMCOMM@ +LAUNCHMON_STATIC_LIBS = @LAUNCHMON_STATIC_LIBS@ +LD = @LD@ +LDFLAGS = @LDFLAGS@ +LIBOBJS = @LIBOBJS@ +LIBS = @LIBS@ +LIBTOOL = @LIBTOOL@ +LIPO = @LIPO@ +LN_S = @LN_S@ +LTLIBOBJS = @LTLIBOBJS@ +MAKEINFO = @MAKEINFO@ +MKDIR_P = @MKDIR_P@ +MUNGE_CFLAGS = @MUNGE_CFLAGS@ +MUNGE_LIBS = @MUNGE_LIBS@ +NM = @NM@ +NMEDIT = @NMEDIT@ +OBJDUMP = @OBJDUMP@ +OBJEXT = @OBJEXT@ +OTOOL = @OTOOL@ +OTOOL64 = @OTOOL64@ +PACKAGE = @PACKAGE@ +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_STRING = @PACKAGE_STRING@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +PATH_SEPARATOR = @PATH_SEPARATOR@ +PKG_CONFIG = @PKG_CONFIG@ +RANLIB = @RANLIB@ +SED = @SED@ +SET_MAKE = @SET_MAKE@ +SHELL = @SHELL@ +STRIP = @STRIP@ +VERSION = @VERSION@ +abs_builddir = @abs_builddir@ +abs_srcdir = @abs_srcdir@ +abs_top_builddir = @abs_top_builddir@ +abs_top_srcdir = @abs_top_srcdir@ +ac_ct_CC = @ac_ct_CC@ +ac_ct_CXX = @ac_ct_CXX@ +ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ +am__include = @am__include@ +am__leading_dot = @am__leading_dot@ +am__quote = @am__quote@ +am__tar = @am__tar@ +am__untar = @am__untar@ +bindir = @bindir@ +build = @build@ +build_alias = @build_alias@ +build_cpu = @build_cpu@ +build_os = @build_os@ +build_vendor = @build_vendor@ +builddir = @builddir@ +datadir = @datadir@ +datarootdir = @datarootdir@ +docdir = @docdir@ +dvidir = @dvidir@ +exec_prefix = @exec_prefix@ +host = @host@ +host_alias = @host_alias@ +host_cpu = @host_cpu@ +host_os = @host_os@ +host_vendor = @host_vendor@ +htmldir = @htmldir@ +includedir = @includedir@ +infodir = @infodir@ +install_sh = @install_sh@ +libdir = @libdir@ +libexecdir = @libexecdir@ +localedir = @localedir@ +localstatedir = @localstatedir@ +lt_ECHO = @lt_ECHO@ +mandir = @mandir@ +mkdir_p = @mkdir_p@ +oldincludedir = @oldincludedir@ +pdfdir = @pdfdir@ +prefix = @prefix@ +program_transform_name = @program_transform_name@ +psdir = @psdir@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +srcdir = @srcdir@ +sysconfdir = @sysconfdir@ +target = @target@ +target_alias = @target_alias@ +target_cpu = @target_cpu@ +target_os = @target_os@ +target_vendor = @target_vendor@ +top_build_prefix = @top_build_prefix@ +top_builddir = @top_builddir@ +top_srcdir = @top_srcdir@ +noinst_LTLIBRARIES = libhostbin.la +libhostbin_la_SOURCES = launch_hostbin.cc +libhostbin_la_CPPFLAGS = -I$(top_srcdir)/../logging -I$(top_srcdir)/../include -DLIBEXEC=\"${pkglibexecdir}\" +all: all-am + +.SUFFIXES: +.SUFFIXES: .cc .lo .o .obj +$(srcdir)/Makefile.in: $(srcdir)/Makefile.am $(am__configure_deps) + @for dep in $?; do \ + case '$(am__configure_deps)' in \ + *$$dep*) \ + ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ + && { if test -f $@; then exit 0; else break; fi; }; \ + exit 1;; \ + esac; \ + done; \ + echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign hostbin/Makefile'; \ + $(am__cd) $(top_srcdir) && \ + $(AUTOMAKE) --foreign hostbin/Makefile +.PRECIOUS: Makefile +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + @case '$?' in \ + *config.status*) \ + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ + *) \ + echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe)'; \ + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe);; \ + esac; + +$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh + +$(top_srcdir)/configure: $(am__configure_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(ACLOCAL_M4): $(am__aclocal_m4_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(am__aclocal_m4_deps): + +clean-noinstLTLIBRARIES: + -test -z "$(noinst_LTLIBRARIES)" || rm -f $(noinst_LTLIBRARIES) + @list='$(noinst_LTLIBRARIES)'; for p in $$list; do \ + dir="`echo $$p | sed -e 's|/[^/]*$$||'`"; \ + test "$$dir" != "$$p" || dir=.; \ + echo "rm -f \"$${dir}/so_locations\""; \ + rm -f "$${dir}/so_locations"; \ + done +libhostbin.la: $(libhostbin_la_OBJECTS) $(libhostbin_la_DEPENDENCIES) + $(AM_V_CXXLD)$(CXXLINK) $(libhostbin_la_OBJECTS) $(libhostbin_la_LIBADD) $(LIBS) + +mostlyclean-compile: + -rm -f *.$(OBJEXT) + +distclean-compile: + -rm -f *.tab.c + +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libhostbin_la-launch_hostbin.Plo@am__quote@ + +.cc.o: +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXXCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@am__fastdepCXX_FALSE@ $(AM_V_CXX) @AM_BACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXXCOMPILE) -c -o $@ $< + +.cc.obj: +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXXCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@am__fastdepCXX_FALSE@ $(AM_V_CXX) @AM_BACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXXCOMPILE) -c -o $@ `$(CYGPATH_W) '$<'` + +.cc.lo: +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LTCXXCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo +@am__fastdepCXX_FALSE@ $(AM_V_CXX) @AM_BACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(LTCXXCOMPILE) -c -o $@ $< + +libhostbin_la-launch_hostbin.lo: launch_hostbin.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libhostbin_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libhostbin_la-launch_hostbin.lo -MD -MP -MF $(DEPDIR)/libhostbin_la-launch_hostbin.Tpo -c -o libhostbin_la-launch_hostbin.lo `test -f 'launch_hostbin.cc' || echo '$(srcdir)/'`launch_hostbin.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libhostbin_la-launch_hostbin.Tpo $(DEPDIR)/libhostbin_la-launch_hostbin.Plo +@am__fastdepCXX_FALSE@ $(AM_V_CXX) @AM_BACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='launch_hostbin.cc' object='libhostbin_la-launch_hostbin.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libhostbin_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libhostbin_la-launch_hostbin.lo `test -f 'launch_hostbin.cc' || echo '$(srcdir)/'`launch_hostbin.cc + +mostlyclean-libtool: + -rm -f *.lo + +clean-libtool: + -rm -rf .libs _libs + +ID: $(HEADERS) $(SOURCES) $(LISP) $(TAGS_FILES) + list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | \ + $(AWK) '{ files[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in files) print i; }; }'`; \ + mkid -fID $$unique +tags: TAGS + +TAGS: $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \ + $(TAGS_FILES) $(LISP) + set x; \ + here=`pwd`; \ + list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | \ + $(AWK) '{ files[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in files) print i; }; }'`; \ + shift; \ + if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \ + test -n "$$unique" || unique=$$empty_fix; \ + if test $$# -gt 0; then \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + "$$@" $$unique; \ + else \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + $$unique; \ + fi; \ + fi +ctags: CTAGS +CTAGS: $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \ + $(TAGS_FILES) $(LISP) + list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | \ + $(AWK) '{ files[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in files) print i; }; }'`; \ + test -z "$(CTAGS_ARGS)$$unique" \ + || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ + $$unique + +GTAGS: + here=`$(am__cd) $(top_builddir) && pwd` \ + && $(am__cd) $(top_srcdir) \ + && gtags -i $(GTAGS_ARGS) "$$here" + +distclean-tags: + -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags + +distdir: $(DISTFILES) + @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + list='$(DISTFILES)'; \ + dist_files=`for file in $$list; do echo $$file; done | \ + sed -e "s|^$$srcdirstrip/||;t" \ + -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ + case $$dist_files in \ + */*) $(MKDIR_P) `echo "$$dist_files" | \ + sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ + sort -u` ;; \ + esac; \ + for file in $$dist_files; do \ + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ + if test -d $$d/$$file; then \ + dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ + if test -d "$(distdir)/$$file"; then \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ + cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ + else \ + test -f "$(distdir)/$$file" \ + || cp -p $$d/$$file "$(distdir)/$$file" \ + || exit 1; \ + fi; \ + done +check-am: all-am +check: check-am +all-am: Makefile $(LTLIBRARIES) +installdirs: +install: install-am +install-exec: install-exec-am +install-data: install-data-am +uninstall: uninstall-am + +install-am: all-am + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am + +installcheck: installcheck-am +install-strip: + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + `test -z '$(STRIP)' || \ + echo "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'"` install +mostlyclean-generic: + +clean-generic: + +distclean-generic: + -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) + -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) + +maintainer-clean-generic: + @echo "This command is intended for maintainers to use" + @echo "it deletes files that may require special tools to rebuild." +clean: clean-am + +clean-am: clean-generic clean-libtool clean-noinstLTLIBRARIES \ + mostlyclean-am + +distclean: distclean-am + -rm -rf ./$(DEPDIR) + -rm -f Makefile +distclean-am: clean-am distclean-compile distclean-generic \ + distclean-tags + +dvi: dvi-am + +dvi-am: + +html: html-am + +html-am: + +info: info-am + +info-am: + +install-data-am: + +install-dvi: install-dvi-am + +install-dvi-am: + +install-exec-am: + +install-html: install-html-am + +install-html-am: + +install-info: install-info-am + +install-info-am: + +install-man: + +install-pdf: install-pdf-am + +install-pdf-am: + +install-ps: install-ps-am + +install-ps-am: + +installcheck-am: + +maintainer-clean: maintainer-clean-am + -rm -rf ./$(DEPDIR) + -rm -f Makefile +maintainer-clean-am: distclean-am maintainer-clean-generic + +mostlyclean: mostlyclean-am + +mostlyclean-am: mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool + +pdf: pdf-am + +pdf-am: + +ps: ps-am + +ps-am: + +uninstall-am: + +.MAKE: install-am install-strip + +.PHONY: CTAGS GTAGS all all-am check check-am clean clean-generic \ + clean-libtool clean-noinstLTLIBRARIES ctags distclean \ + distclean-compile distclean-generic distclean-libtool \ + distclean-tags distdir dvi dvi-am html html-am info info-am \ + install install-am install-data install-data-am install-dvi \ + install-dvi-am install-exec install-exec-am install-html \ + install-html-am install-info install-info-am install-man \ + install-pdf install-pdf-am install-ps install-ps-am \ + install-strip installcheck installcheck-am installdirs \ + maintainer-clean maintainer-clean-generic mostlyclean \ + mostlyclean-compile mostlyclean-generic mostlyclean-libtool \ + pdf pdf-am ps ps-am tags uninstall uninstall-am + + +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/src/fe/hostbin/launch_hostbin.cc b/src/fe/hostbin/launch_hostbin.cc new file mode 100644 index 00000000..6b6c8eed --- /dev/null +++ b/src/fe/hostbin/launch_hostbin.cc @@ -0,0 +1,595 @@ +/* +This file is part of Spindle. For copyright information see the COPYRIGHT +file in the top level directory, or at +https://github.com/hpc/Spindle/blob/master/COPYRIGHT + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License (as published by the Free Software +Foundation) version 2.1 dated February 1999. This program is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY; without even the IMPLIED +WARRANTY OF MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms +and conditions of the GNU General Public License for more details. You should +have received a copy of the GNU Lesser General Public License along with this +program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +#include +#include +#include + +using namespace std; + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "spindle_launch.h" +#include "spindle_debug.h" + +#if !defined(LIBEXEC) +#error Expected libexec to be defined +#endif +static const char libexec_dir[] = LIBEXEC "/"; + +static sigset_t child_blocked_mask; +static sigset_t child_unblocked_mask; +static bool hit_sigchild = 0; + +static bool hostbin_exit_signal = false; +static bool launcher_exit_signal = false; +static bool exit_detected = false; +static pthread_cond_t exited_cvar; +static pthread_mutex_t exited_mut; +static pthread_cond_t result_cvar; +static pthread_mutex_t result_mut; + +static void on_child(int) +{ + hit_sigchild = true; +} + +class IOThread { + static const int RD = 0; + static const int WR = 1; + static const int READ_SIZE = 0x1000; + + int launcher_stdout_pipe[2]; + int launcher_stderr_pipe[2]; + int hostbin_launcher_pipe[2]; + int hostbin_result_pipe[2]; + int done_pipe[2]; + + bool started; + pthread_t thrd; + char read_buffer[READ_SIZE]; + string partial_line; + + bool launcher_done; + bool hostbin_done; + vector hostlist; +private: + void mkpipe(int (&p)[2], bool set_nonblock = false) { + int result = pipe(p); + if (result == -1) { + const char *errstr = strerror(errno); + err_printf("Error creating pipes for IOThread: %s\n", errstr); + fprintf(stderr, "Could not create pipe for startup binary: %s\n", errstr); + exit(-1); + } + if (set_nonblock) { + int orig_flags = fcntl(p[RD], F_GETFL); + fcntl(p[RD], F_SETFL, orig_flags | O_NONBLOCK); + } + } + + bool copyInput(int from_fd, int to_fd1, int to_fd2, bool &read_anything) { + ssize_t result; + read_anything = false; + do { + do { + result = read(from_fd, read_buffer, sizeof(read_buffer)); + } while (result == -1 && errno == EINTR); + if (result == -1) { + char *errstr = strerror(errno); + err_printf("IOThread Failed to read input from %d: %s\n", from_fd, errstr); + return false; + } + if (result > 0) { + read_anything = true; + if (to_fd1 != -1) { + write(to_fd1, read_buffer, result); + } + if (to_fd2 != -1) { + write(to_fd2, read_buffer, result); + } + } + } while (result > 0); + + return true; + } + + void parseHostbinStdout(int fd, bool &read_anything) { + read_anything = false; + ssize_t result; + for(;;) { + do { + result = read(fd, read_buffer, sizeof(read_buffer)); + } while (result == -1 && errno == EINTR); + if (result == 0 || (result == -1 && errno == EAGAIN)) + break; + if (result == -1) { + char *errstr = strerror(errno); + err_printf("IOThread failed to read hostbin input from %d: %s\n", fd, errstr); + return; + } + read_anything = true; + ssize_t cur = 0, buffer_size = result; + while (cur < buffer_size) { + char *str = read_buffer + cur; + unsigned int str_size = 0; + while ((str_size + cur < buffer_size) && + (str[str_size] != '\n') && + (str[str_size] != '\0')) str_size++; + + if (str[str_size] != '\n' && str[str_size] != '\0') { + partial_line = string(read_buffer + cur, str_size); + return; + } + string host; + if (str_size > 0) + host = partial_line + string(read_buffer + cur, str_size); + else + host = partial_line; + cur = cur + str_size + 1; + if (!host.empty()) { + debug_printf3("Adding %s to hostlist from hostbin\n", host.c_str()); + hostlist.push_back(host); + } + if (!partial_line.empty()) + partial_line = string(); + } + } + } + + void handleProcessExit(bool &hostbin_exited, bool &launcher_exited) + { + pthread_mutex_lock(&exited_mut); + assert(!hostbin_exit_signal && !launcher_exit_signal); + exit_detected = true; + pthread_cond_signal(&exited_cvar); + pthread_mutex_unlock(&exited_mut); + + pthread_mutex_lock(&result_mut); + while (!hostbin_exit_signal && !launcher_exit_signal) + pthread_cond_wait(&result_cvar, &result_mut); + hostbin_exited = hostbin_exit_signal; + launcher_exited = launcher_exit_signal; + hostbin_exit_signal = launcher_exit_signal = false; + pthread_mutex_unlock(&result_mut); + } + + void markLauncherDone() + { + launcher_done = true; + close(launcher_stdout_pipe[RD]); + close(launcher_stderr_pipe[RD]); + launcher_stdout_pipe[RD] = launcher_stderr_pipe[WR] = -1; + } + + void markHostbinDone() + { + hostbin_done = true; + close(hostbin_launcher_pipe[WR]); + close(hostbin_result_pipe[RD]); + hostbin_launcher_pipe[WR] = hostbin_result_pipe[RD] = -1; + } + + static void *iothread_main(void *param) + { + IOThread *thr = static_cast(param); + thr->main_loop(); + return NULL; + } + + void main_loop() + { + bool read_anything; + for (;;) { + bool launcher_done = false, hostbin_done = false; + fd_set read_set; + int max_fd = 0; + FD_ZERO(&read_set); + +#define ADD_FD(FD) \ + do { if (FD != -1) { \ + FD_SET(FD, &read_set); \ + if (FD > max_fd) max_fd = FD; \ + } } while (0) + + ADD_FD(launcher_stdout_pipe[RD]); + ADD_FD(launcher_stderr_pipe[RD]); + ADD_FD(hostbin_result_pipe[RD]); + ADD_FD(done_pipe[RD]); + + if (max_fd == 0) + break; + + hit_sigchild = false; + int result = pselect(max_fd+1, &read_set, NULL, NULL, NULL, &child_unblocked_mask); + if (!hit_sigchild && result == -1) { + perror("Error calling pselect"); + exit(-1); + } + + if (result > 0 && FD_ISSET(launcher_stderr_pipe[RD], &read_set)) { + copyInput(launcher_stderr_pipe[RD], 2, hostbin_launcher_pipe[WR], read_anything); + if (!read_anything) + launcher_done = true; + } + if (result > 0 && FD_ISSET(launcher_stdout_pipe[RD], &read_set)) { + copyInput(launcher_stdout_pipe[RD], 1, hostbin_launcher_pipe[WR], read_anything); + if (!read_anything) + launcher_done = true; + } + if (result > 0 && FD_ISSET(hostbin_result_pipe[RD], &read_set)) { + parseHostbinStdout(hostbin_result_pipe[RD], read_anything); + if (!read_anything) + hostbin_done = true; + } + if (result > 0 && FD_ISSET(done_pipe[RD], &read_set)) { + break; + } + if (hit_sigchild) { + handleProcessExit(hostbin_done, launcher_done); + } + if (hostbin_done) { + parseHostbinStdout(hostbin_result_pipe[RD], read_anything); + markHostbinDone(); + continue; + } + if (launcher_done) { + copyInput(launcher_stderr_pipe[RD], 2, hostbin_launcher_pipe[WR], read_anything); + copyInput(launcher_stdout_pipe[RD], 1, hostbin_launcher_pipe[WR], read_anything); + markLauncherDone(); + } + } + } +public: + IOThread() : + started(false), + launcher_done(false), + hostbin_done(false) + { + mkpipe(launcher_stdout_pipe, true); + mkpipe(launcher_stderr_pipe, true); + mkpipe(hostbin_launcher_pipe); + mkpipe(hostbin_result_pipe, true); + mkpipe(done_pipe); + } + + ~IOThread() { + close(launcher_stdout_pipe[0]); + close(launcher_stdout_pipe[1]); + close(launcher_stderr_pipe[0]); + close(launcher_stderr_pipe[1]); + close(hostbin_launcher_pipe[0]); + close(hostbin_launcher_pipe[1]); + close(hostbin_result_pipe[0]); + close(hostbin_result_pipe[1]); + + if (started) { + char c = 'c'; + write(done_pipe[WR], &c, sizeof(c)); + pthread_join(thrd, NULL); + } + + close(done_pipe[0]); + close(done_pipe[1]); + } + + void setupLauncherPipesParent() { + close(launcher_stdout_pipe[WR]); + close(launcher_stderr_pipe[WR]); + launcher_stdout_pipe[WR] = launcher_stderr_pipe[WR] = -1; + } + + void setupLauncherPipesChild() { + dup2(launcher_stdout_pipe[WR], 1); + dup2(launcher_stderr_pipe[WR], 2); + close(launcher_stdout_pipe[RD]); + close(launcher_stderr_pipe[RD]); + } + + void setupHostbinPipesParent() { + close(hostbin_launcher_pipe[RD]); + close(hostbin_result_pipe[WR]); + hostbin_launcher_pipe[RD] = hostbin_result_pipe[WR] = -1; + } + + void setupHostbinPipesChild() { + dup2(hostbin_launcher_pipe[RD], 0); + dup2(hostbin_result_pipe[WR], 1); + close(hostbin_launcher_pipe[WR]); + close(hostbin_result_pipe[RD]); + } + + bool run() { + int result = pthread_create(&thrd, NULL, iothread_main, static_cast(this)); + if (result == -1) { + err_printf("Spindle Error: Failed to create IOThread: %s\n", strerror(errno)); + return false; + } + started = true; + return true; + } + + bool launcherDone() { + return launcher_done; + } + + vector &getHosts() { + return hostlist; + } +}; + +class ProcManager +{ +private: + enum proc_result_t { + not_exited, + proc_success, + proc_error + }; + + IOThread *io; + pid_t launcher_pid; + pid_t hostbin_pid; + bool launcher_running; + bool hostbin_running; + proc_result_t launcher_result; + proc_result_t hostbin_result; + int launcher_return; + + proc_result_t collectHostbinResult() { + int status, result; + result = waitpid(hostbin_pid, &status, WNOHANG); + if (result == 0) { + return not_exited; + } + if (result == -1) { + err_printf("Unexpected error calling waitpid for hostbin: %s\n", strerror(errno)); + return proc_error; + } + if (WIFEXITED(status) && WEXITSTATUS(status) == 0) { + debug_printf("Hostbin completed successfully\n"); + hostbin_running = false; + return proc_success; + } + else if (WIFSIGNALED(status)) { + err_printf("hostbin exited with signal %d\n", WTERMSIG(status)); + hostbin_running = false; + return proc_error; + } + else if (WIFEXITED(status) && WEXITSTATUS(status) != 0) { + err_printf("hostbin exited with non-zero return %d\n", WEXITSTATUS(status)); + hostbin_running = false; + return proc_error; + } + else { + err_printf("Unexpected return from waitpid on hostbin: %d\n", status); + return proc_error; + } + } + + proc_result_t collectLauncherResult() { + int status, result; + result = waitpid(launcher_pid, &status, WNOHANG); + if (result == 0) { + return not_exited; + } + + if (result == -1) { + err_printf("Unexpected error calling waitpid for launcher: %s\n", strerror(errno)); + return proc_error; + } + if (WIFEXITED(status)) { + debug_printf("Launcher returned with return code %d\n", WEXITSTATUS(status)); + launcher_return = WEXITSTATUS(status); + launcher_running = false; + return proc_success; + } + else if (WIFSIGNALED(status)) { + err_printf("Launcher exited with signal %d\n", WTERMSIG(status)); + launcher_return = -1; + launcher_running = false; + return proc_success; + } + else { + err_printf("waitpid returned unexpected status for launcher: %d\n", status); + return proc_error; + } + } +public: + ProcManager(IOThread *io_) : + io(io_), + launcher_pid(-1), + hostbin_pid(-1), + launcher_running(false), + hostbin_running(false), + launcher_result(not_exited), + hostbin_result(not_exited), + launcher_return(0) + { + } + + void runLauncher(int /*launcher_argc*/, char **launcher_argv) { + launcher_pid = fork(); + if (launcher_pid == -1) { + fprintf(stderr, "Spindle error: Failed to fork process for job launcher: %s\n", strerror(errno)); + exit(-1); + } + if (launcher_pid == 0) { + io->setupLauncherPipesChild(); + execvp(launcher_argv[0], launcher_argv); + char *errstr = strerror(errno); + fprintf(stderr, "Spindle error: Could not invoke job launcher %s: %s\n", launcher_argv[0], errstr); + exit(-1); + } + io->setupLauncherPipesParent(); + launcher_running = true; + } + + void runHostbin(string hostbin) { + char launcher_pid_str[32]; + char *args[3]; + + assert(launcher_pid != -1); + + hostbin_pid = fork(); + if (launcher_pid == -1) { + fprintf(stderr, "Spindle error: Failed to fork process for hostbin: %s\n", strerror(errno)); + exit(-1); + } + if (hostbin_pid == 0) { + snprintf(launcher_pid_str, sizeof(launcher_pid_str), "%d", launcher_pid); + args[0] = const_cast(hostbin.c_str()); + args[1] = launcher_pid_str; + args[2] = NULL; + io->setupHostbinPipesChild(); + execvp(args[0], args); + + fprintf(stderr, "Spindle failed to exec hostbin %s: %s\n", args[0], strerror(errno)); + kill(launcher_pid, SIGTERM); + exit(-1); + } + io->setupHostbinPipesParent(); + hostbin_running = true; + } + + bool waitForResult() { + bool hb_exited = false, la_exited = false; + + pthread_mutex_lock(&exited_mut); + while (!exit_detected) + pthread_cond_wait(&exited_cvar, &exited_mut); + pthread_mutex_unlock(&exited_mut); + + if (hostbin_running) { + hostbin_result = collectHostbinResult(); + hb_exited = (hostbin_result != not_exited); + } + if (launcher_running) { + launcher_result = collectLauncherResult(); + la_exited = (launcher_result != not_exited); + } + + pthread_mutex_lock(&result_mut); + hostbin_exit_signal = hb_exited; + launcher_exit_signal = la_exited; + pthread_cond_signal(&result_cvar); + pthread_mutex_unlock(&result_mut); + + return true; + } + + bool launcherDone() { + return launcher_result != not_exited; + } + + int launcherResult() { + return launcher_return; + } + + bool hostbinDone() { + return hostbin_result != not_exited; + } + + bool hostbinError() { + return hostbin_result == proc_error; + } + + +}; + +static bool fileExists(string f) +{ + struct stat buf; + int result = stat(f.c_str(), &buf); + return result != -1; +} + +int startHostbinFE(string hostbin_exe, + int app_argc, char **app_argv, + spindle_args_t *params) +{ + IOThread io_thread; + ProcManager proc_manager(&io_thread); + + if (hostbin_exe.find('/') == string::npos) { + string fullpath = string(libexec_dir) + hostbin_exe; + if (fileExists(fullpath)) + hostbin_exe = fullpath; + } + + signal(SIGPIPE, SIG_IGN); + signal(SIGCHLD, on_child); + pthread_sigmask(SIG_BLOCK, NULL, &child_blocked_mask); + pthread_sigmask(SIG_BLOCK, NULL, &child_unblocked_mask); + sigaddset(&child_blocked_mask, SIGCHLD); + sigdelset(&child_unblocked_mask, SIGCHLD); + pthread_sigmask(SIG_BLOCK, &child_blocked_mask, NULL); + pthread_cond_init(&exited_cvar, NULL); + pthread_mutex_init(&exited_mut, NULL); + pthread_cond_init(&result_cvar, NULL); + pthread_mutex_init(&result_mut, NULL); + + + proc_manager.runLauncher(app_argc, app_argv); + proc_manager.runHostbin(hostbin_exe); + + io_thread.run(); + + proc_manager.waitForResult(); + + if (proc_manager.launcherDone()) { + err_printf("Job launcher terminated prematurely\n"); + return proc_manager.launcherResult(); + } + if (proc_manager.hostbinError()) { + err_printf("Hostbin exited with an error code. Exiting\n"); + return -1; + } + assert(proc_manager.hostbinDone()); + + vector &hostvec = io_thread.getHosts(); + if (hostvec.empty()) { + err_printf("Empty hostlist from hostbin %s\n", hostbin_exe.c_str()); + return -1; + } + + char **host_array = (char **) malloc(sizeof(char *) * (hostvec.size()+1)); + for (unsigned int i = 0; i < hostvec.size(); i++) + host_array[i] = const_cast(hostvec[i].c_str()); + host_array[hostvec.size()] = NULL; + + int result = spindleInitFE(const_cast(host_array), params); + if (result == -1) { + debug_printf("spindleInitFE returned an error\n"); + return -1; + } + + proc_manager.waitForResult(); + assert(proc_manager.launcherDone()); + return proc_manager.launcherResult(); +} diff --git a/src/fe/launchmon/spindle_fe_lmon.cc b/src/fe/launchmon/spindle_fe_lmon.cc index 44c7a42b..08431af1 100644 --- a/src/fe/launchmon/spindle_fe_lmon.cc +++ b/src/fe/launchmon/spindle_fe_lmon.cc @@ -191,7 +191,6 @@ int startLaunchmonFE(int app_argc, char *app_argv[], err_printf("[LMON FE] Error getting process table\n"); return EXIT_FAILURE; } - result = spindleInitFE(hosts, params); if (result == -1) { diff --git a/src/fe/openmpi_intercept/ompi_intercept.c b/src/fe/openmpi_intercept/ompi_intercept.c index a2196f58..c6883f61 100644 --- a/src/fe/openmpi_intercept/ompi_intercept.c +++ b/src/fe/openmpi_intercept/ompi_intercept.c @@ -140,7 +140,7 @@ static void on_load() exit(-1); } - result = sscanf(args, "%255s ; %lx ;%lx ; %lx", filename, &bp_addr, &ptable_addr, &ptable_size_addr); + result = sscanf(args, "%255s %lx %lx %lx", filename, &bp_addr, &ptable_addr, &ptable_size_addr); if (result != 4) { fprintf(stderr, "Spindle error in mpiexec intercept: Could not parse SPINDLE_OMPI_INTERCEPT\n"); fprintf(stderr, "result = %d, args = %s, filename = %s\n", result, args, filename); diff --git a/src/fe/openmpi_intercept/parse_openmpi.cc b/src/fe/openmpi_intercept/parse_openmpi.cc index bb4f7f74..a0825aab 100644 --- a/src/fe/openmpi_intercept/parse_openmpi.cc +++ b/src/fe/openmpi_intercept/parse_openmpi.cc @@ -272,9 +272,9 @@ bool setOpenMPIInterceptEnv(string launcher_rel) last_slash = last_slash ? last_slash+1 : real_launcher; stringstream ss; - ss << last_slash << " ; " << std::hex << - parser->getMPIRBreakpointAddr() << ";" << - parser->getMPIRProctabAddr() << ";" << + ss << last_slash << " " << std::hex << + parser->getMPIRBreakpointAddr() << " " << + parser->getMPIRProctabAddr() << " " << parser->getMPIRProctabSizeAddr(); setenv("SPINDLE_OMPI_INTERCEPT", ss.str().c_str(), 1); diff --git a/src/fe/startup/Makefile.am b/src/fe/startup/Makefile.am index 7f50980b..99281c74 100644 --- a/src/fe/startup/Makefile.am +++ b/src/fe/startup/Makefile.am @@ -5,7 +5,7 @@ AM_CPPFLAGS = -I$(top_srcdir)/../logging spindle_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_srcdir)/../include -I$(top_srcdir)/comlib -I$(top_srcdir)/../server/cache -I$(top_srcdir)/../server/comlib -I$(top_srcdir)/../utils -I$(top_srcdir)/../cobo -DBINDIR=\"$(pkglibexecdir)\" -DLIBEXECDIR=\"$(pkglibexecdir)\" spindle_SOURCES = spindle_fe_main.cc spindle_fe_serial.cc spindle_fe.cc parse_launcher.cc parse_launcher_args.cc parseargs.cc parse_preload.cc $(top_srcdir)/../utils/pathfn.c $(top_srcdir)/../utils/keyfile.c $(top_srcdir)/../utils/parseloc.c -spindle_LDADD = $(top_builddir)/logging/libspindlelogc.la $(top_builddir)/openmpi_intercept/libparseompi.la +spindle_LDADD = $(top_builddir)/logging/libspindlelogc.la $(top_builddir)/openmpi_intercept/libparseompi.la $(top_builddir)/hostbin/libhostbin.la if LMON spindle_LDADD += $(top_builddir)/launchmon/libfelmon.la diff --git a/src/fe/startup/Makefile.in b/src/fe/startup/Makefile.in index edb25049..5a16b26f 100644 --- a/src/fe/startup/Makefile.in +++ b/src/fe/startup/Makefile.in @@ -65,8 +65,9 @@ spindle_OBJECTS = $(am_spindle_OBJECTS) am__DEPENDENCIES_1 = spindle_DEPENDENCIES = $(top_builddir)/logging/libspindlelogc.la \ $(top_builddir)/openmpi_intercept/libparseompi.la \ - $(am__append_1) $(am__append_2) $(am__append_3) \ - $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) + $(top_builddir)/hostbin/libhostbin.la $(am__append_1) \ + $(am__append_2) $(am__append_3) $(am__DEPENDENCIES_1) \ + $(am__DEPENDENCIES_1) AM_V_lt = $(am__v_lt_$(V)) am__v_lt_ = $(am__v_lt_$(AM_DEFAULT_VERBOSITY)) am__v_lt_0 = --silent @@ -252,8 +253,8 @@ spindle_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_srcdir)/../include -I$(top_srcdir)/com spindle_SOURCES = spindle_fe_main.cc spindle_fe_serial.cc spindle_fe.cc parse_launcher.cc parse_launcher_args.cc parseargs.cc parse_preload.cc $(top_srcdir)/../utils/pathfn.c $(top_srcdir)/../utils/keyfile.c $(top_srcdir)/../utils/parseloc.c spindle_LDADD = $(top_builddir)/logging/libspindlelogc.la \ $(top_builddir)/openmpi_intercept/libparseompi.la \ - $(am__append_1) $(am__append_2) $(am__append_3) $(MUNGE_LIBS) \ - $(GCRYPT_LIBS) + $(top_builddir)/hostbin/libhostbin.la $(am__append_1) \ + $(am__append_2) $(am__append_3) $(MUNGE_LIBS) $(GCRYPT_LIBS) all: all-am .SUFFIXES: diff --git a/src/fe/startup/parse_launcher.cc b/src/fe/startup/parse_launcher.cc index 07aa2062..f05ebd51 100644 --- a/src/fe/startup/parse_launcher.cc +++ b/src/fe/startup/parse_launcher.cc @@ -168,11 +168,14 @@ LauncherParser *CmdLineParser::getParser() } ModifyArgv::ModifyArgv(int argc_, char **argv_, + int daemon_argc_, char **daemon_argv_, spindle_args_t *params_) : argc(argc_), argv(argv_), new_argc(0), new_argv(NULL), + daemon_argc(daemon_argc_), + daemon_argv(daemon_argv_), params(params_), parser(NULL) { @@ -274,7 +277,11 @@ void ModifyArgv::modifyCmdLine() snprintf(number_str, 32, "%u", params->number); string number(number_str); - new_argv = (char **) malloc(sizeof(char *) * (argc + 5)); + char daemon_argc_str[32]; + snprintf(daemon_argc_str, 32, "%u", daemon_argc); + + int new_argv_size = argc + 7 + daemon_argc; + new_argv = (char **) malloc(sizeof(char *) * new_argv_size); int n = 0; int p = parser->launcherAt(); @@ -290,6 +297,13 @@ void ModifyArgv::modifyCmdLine() new_argv[n++] = bg_env; #else new_argv[n++] = strdup(spindle_bootstrap); + if (params->startup_type == startup_hostbin) { + new_argv[n++] = strdup("-daemon_args"); + new_argv[n++] = strdup(daemon_argc_str); + for (int k = 0; k < daemon_argc; k++) { + new_argv[n++] = strdup(daemon_argv[k]); + } + } new_argv[n++] = strdup(location.c_str()); new_argv[n++] = strdup(number.c_str()); new_argv[n++] = strdup(options.c_str()); @@ -301,6 +315,7 @@ void ModifyArgv::modifyCmdLine() new_argv[n++] = strdup(argv[p]); } new_argv[n] = NULL; + assert(n < new_argv_size); new_argc = n; } @@ -340,7 +355,7 @@ void ModifyArgv::getNewArgv(int &newargc, char** &newargv) newargc = new_argc; newargv = new_argv; - if (params->use_launcher == openmpi_launcher) { + if (params->use_launcher == openmpi_launcher && params->startup_type == startup_lmon) { setOpenMPIInterceptEnv(argv[parser->launcherAt()]); } } diff --git a/src/fe/startup/parse_launcher.h b/src/fe/startup/parse_launcher.h index 1015bfdd..30fcc4c7 100644 --- a/src/fe/startup/parse_launcher.h +++ b/src/fe/startup/parse_launcher.h @@ -35,6 +35,8 @@ class ModifyArgv { char **argv; int new_argc; char **new_argv; + int daemon_argc; + char **daemon_argv; spindle_args_t *params; CmdLineParser *parser; @@ -45,6 +47,7 @@ class ModifyArgv { public: ModifyArgv(int argc, char **argv, + int daemon_argc, char **daemon_argv, spindle_args_t *params); void getNewArgv(int &newargc, char** &newargv); }; diff --git a/src/fe/startup/parseargs.cc b/src/fe/startup/parseargs.cc index fb5097c7..fb323e35 100644 --- a/src/fe/startup/parseargs.cc +++ b/src/fe/startup/parseargs.cc @@ -57,6 +57,7 @@ using namespace std; #define OPENMPI 271 #define WRECK 272 #define NOMPI 273 +#define HOSTBIN 274 #define GROUP_RELOC 1 #define GROUP_PUSHPULL 2 @@ -92,6 +93,13 @@ static const unsigned long default_pushpull_opts = OPT_PUSH; static const unsigned long default_misc_opts = OPT_STRIP; static const unsigned long default_sec = DEFAULT_SEC; +#if defined(HOSTBIN_PATH) +static char default_hostbin_path[] = HOSTBIN_PATH; +static char *hostbin_path = default_hostbin_path; +#else +static char *hostbin_path = NULL; +#endif + static unsigned long enabled_opts = 0; static unsigned long disabled_opts = 0; @@ -102,6 +110,7 @@ static bool done = false; static bool hide_fd = true; static int sec_model = -1; static int launcher = 0; +static int startup_type = 0; static set python_prefixes; static const char *default_python_prefixes = PYTHON_INST_PREFIX; @@ -187,6 +196,8 @@ struct argp_option options[] = { "Colon-seperated list of directories that contain the python install location", GROUP_MISC }, { "debug", DEBUG, YESNO, 0, "If yes, hide spindle from debuggers so they think libraries come from the original locations. May cause extra overhead. Default: no", GROUP_MISC }, + { "hostbin", HOSTBIN, "EXECUTABLE", 0, + "Path to a script that returns the hostlist for a job on a cluster", GROUP_MISC }, { "preload", PRELOAD, "FILE", 0, "Provides a text file containing a white-space separated list of files that should be " "relocated to each node before execution begins", GROUP_MISC }, @@ -312,6 +323,10 @@ static int parse(int key, char *arg, struct argp_state *vstate) user_python_prefixes = arg; return 0; } + else if (key == HOSTBIN) { + hostbin_path = arg; + return 0; + } else if (key == ARGP_KEY_ARGS) { mpi_argv = state->argv + state->next; mpi_argc = state->argc - state->next; @@ -340,6 +355,14 @@ static int parse(int key, char *arg, struct argp_state *vstate) /* Set any reloc options */ opts |= all_reloc_opts & ~disabled_opts & (enabled_opts | default_reloc_opts); + /* Set startup type */ + if (launcher == serial_launcher) + startup_type = startup_serial; + else if (hostbin_path != NULL) + startup_type = startup_hostbin; + else + startup_type = startup_lmon; + /* Set security options */ if (sec_model == -1) sec_model = default_sec; @@ -460,3 +483,13 @@ int getLauncher() { return launcher; } + +std::string getHostbin() +{ + return string(hostbin_path); +} + +int getStartupType() +{ + return startup_type; +} diff --git a/src/fe/startup/parseargs.h b/src/fe/startup/parseargs.h index 2253cb01..a1eae759 100644 --- a/src/fe/startup/parseargs.h +++ b/src/fe/startup/parseargs.h @@ -24,6 +24,8 @@ char *getPreloadFile(); unsigned int getPort(); std::string getLocation(int number); std::string getPythonPrefixes(); +std::string getHostbin(); +int getStartupType(); bool isLoggingEnabled(); int getLauncher(); bool hideFDs(); diff --git a/src/fe/startup/spindle_fe.cc b/src/fe/startup/spindle_fe.cc index cb15af8c..232bf93f 100644 --- a/src/fe/startup/spindle_fe.cc +++ b/src/fe/startup/spindle_fe.cc @@ -47,7 +47,7 @@ void pack_param(char *value, char *buffer, unsigned int &pos) static int pack_data(spindle_args_t *args, void* &buffer, unsigned &buffer_size) { - buffer_size = sizeof(unsigned int) * 5; + buffer_size = sizeof(unsigned int) * 6; buffer_size += args->location ? strlen(args->location) + 1 : 1; buffer_size += args->pythonprefix ? strlen(args->pythonprefix) + 1 : 1; buffer_size += args->preloadfile ? strlen(args->preloadfile) + 1 : 1; @@ -59,6 +59,7 @@ static int pack_data(spindle_args_t *args, void* &buffer, unsigned &buffer_size) pack_param(args->opts, buf, pos); pack_param(args->shared_secret, buf, pos); pack_param(args->use_launcher, buf, pos); + pack_param(args->startup_type, buf, pos); pack_param(args->location, buf, pos); pack_param(args->pythonprefix, buf, pos); pack_param(args->preloadfile, buf, pos); diff --git a/src/fe/startup/spindle_fe_main.cc b/src/fe/startup/spindle_fe_main.cc index 615713a3..37e78000 100644 --- a/src/fe/startup/spindle_fe_main.cc +++ b/src/fe/startup/spindle_fe_main.cc @@ -37,7 +37,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA using namespace std; static void setupLogging(int argc, char **argv); -static void getAppCommandLine(int argc, char *argv[], spindle_args_t *args, int *mod_argc, char ***mod_argv); +static void getAppCommandLine(int argc, char *argv[], int daemon_argc, char *daemon_argv[], spindle_args_t *args, int *mod_argc, char ***mod_argv); static void getDaemonCommandLine(int *daemon_argc, char ***daemon_argv, spindle_args_t *args); static void parseCommandLine(int argc, char *argv[], spindle_args_t *args); static void logUser(); @@ -59,6 +59,10 @@ extern int startSerialFE(int app_argc, char *app_argv[], int daemon_argc, char *daemon_argv[], spindle_args_t *params); +extern int startHostbinFE(string hostbin_exe, + int app_argc, char **app_argv, + spindle_args_t *params); + int main(int argc, char *argv[]) { int result; @@ -71,15 +75,6 @@ int main(int argc, char *argv[]) if (isLoggingEnabled()) logUser(); - int app_argc; - char **app_argv; - getAppCommandLine(argc, argv, ¶ms, &app_argc, &app_argv); - debug_printf2("Application CmdLine: "); - for (int i = 0; i < app_argc; i++) { - bare_printf2("%s ", app_argv[i]); - } - bare_printf2("\n"); - setupSecurity(¶ms); int daemon_argc; @@ -91,9 +86,21 @@ int main(int argc, char *argv[]) } bare_printf2("\n"); - if (params.use_launcher == serial_launcher) + int app_argc; + char **app_argv; + getAppCommandLine(argc, argv, daemon_argc, daemon_argv, ¶ms, &app_argc, &app_argv); + debug_printf2("Application CmdLine: "); + for (int i = 0; i < app_argc; i++) { + bare_printf2("%s ", app_argv[i]); + } + bare_printf2("\n"); + + if (params.use_launcher == serial_launcher) { + debug_printf("Starting application in serial mode\n"); result = startSerialFE(app_argc, app_argv, daemon_argc, daemon_argv, ¶ms); - else { + } + else if (params.startup_type == startup_lmon) { + debug_printf("Starting application with launchmon\n"); #if defined(HAVE_LMON) result = startLaunchmonFE(app_argc, app_argv, daemon_argc, daemon_argv, ¶ms); #else @@ -102,6 +109,11 @@ int main(int argc, char *argv[]) return -1; #endif } + else if (params.startup_type == startup_hostbin) { + debug_printf("Starting application with hostbin\n"); + result = startHostbinFE(getHostbin(), + app_argc, app_argv, ¶ms); + } if (OPT_GET_SEC(params.opts) == OPT_SEC_KEYFILE) { clean_keyfile(params.number); @@ -120,6 +132,7 @@ static void parseCommandLine(int argc, char *argv[], spindle_args_t *args) args->opts = opts; args->shared_secret = get_shared_secret(); args->use_launcher = getLauncher(); + args->startup_type = getStartupType(); args->location = strdup(getLocation(args->number).c_str()); args->pythonprefix = strdup(getPythonPrefixes().c_str()); args->preloadfile = getPreloadFile(); @@ -139,7 +152,10 @@ static void setupLogging(int argc, char **argv) static unsigned int get_shared_secret() { - unsigned int shared_secret; + static unsigned int shared_secret = 0; + if (shared_secret != 0) + return shared_secret; + int fd = open("/dev/urandom", O_RDONLY); if (fd == -1) fd = open("/dev/random", O_RDONLY); @@ -159,14 +175,14 @@ static unsigned int get_shared_secret() } -static void getAppCommandLine(int argc, char *argv[], spindle_args_t *params, int *mod_argc, char ***mod_argv) +static void getAppCommandLine(int argc, char *argv[], int daemon_argc, char *daemon_argv[], spindle_args_t *params, int *mod_argc, char ***mod_argv) { int app_argc; char **app_argv; getAppArgs(&app_argc, &app_argv); - ModifyArgv modargv(app_argc, app_argv, params); + ModifyArgv modargv(app_argc, app_argv, daemon_argc, daemon_argv, params); modargv.getNewArgv(*mod_argc, *mod_argv); } @@ -176,9 +192,11 @@ static void getAppCommandLine(int argc, char *argv[], spindle_args_t *params, in char spindle_daemon[] = LIBEXECDIR "/spindle_be"; char spindle_serial_arg[] = "--spindle_serial"; char spindle_lmon_arg[] = "--spindle_lmon"; +char spindle_hostbin_arg[] = "--spindle_hostbin"; + static void getDaemonCommandLine(int *daemon_argc, char ***daemon_argv, spindle_args_t *args) { - char **daemon_opts = (char **) malloc(8 * sizeof(char *)); + char **daemon_opts = (char **) malloc(10 * sizeof(char *)); char number_s[32]; int i = 0; @@ -190,8 +208,10 @@ static void getDaemonCommandLine(int *daemon_argc, char ***daemon_argv, spindle_ daemon_opts[i++] = spindle_daemon; if (args->use_launcher == serial_launcher) daemon_opts[i++] = spindle_serial_arg; - else + else if (args->startup_type == startup_lmon) daemon_opts[i++] = spindle_lmon_arg; + else if (args->startup_type == startup_hostbin) + daemon_opts[i++] = spindle_hostbin_arg; #define STR2(X) #X #define STR(X) STR2(X) @@ -204,7 +224,14 @@ static void getDaemonCommandLine(int *daemon_argc, char ***daemon_argv, spindle_ } daemon_opts[i++] = strdup(number_s); - daemon_opts[i++] = NULL; + if (args->startup_type == startup_hostbin) { + char port_str[32], ss_str[32]; + snprintf(port_str, 32, "%d", getPort()); + snprintf(ss_str, 32, "%d", get_shared_secret()); + daemon_opts[i++] = strdup(port_str); + daemon_opts[i++] = strdup(ss_str); + } + daemon_opts[i] = NULL; *daemon_argc = i; *daemon_argv = daemon_opts; diff --git a/src/include/spindle_launch.h b/src/include/spindle_launch.h index 54e802a0..32bad673 100644 --- a/src/include/spindle_launch.h +++ b/src/include/spindle_launch.h @@ -51,6 +51,11 @@ extern "C" { #define wreckrun_launcher (1 << 3) #define marker_launcher (1 << 4) +/* Possible values for startup_type */ +#define startup_serial 0 +#define startup_lmon 1 +#define startup_hostbin 2 + /* Parameters for configuring Spindle */ typedef struct { /* A unique number that will be used to identify this spindle session */ @@ -69,6 +74,9 @@ typedef struct { /* The type of the MPI launcher */ unsigned int use_launcher; + /* The mechanism used to start Spindle daemons */ + unsigned int startup_type; + /* The local-disk location where Spindle will store its cache */ char *location; diff --git a/src/server/config.h.in b/src/server/config.h.in index bdf5614b..45659a66 100644 --- a/src/server/config.h.in +++ b/src/server/config.h.in @@ -60,6 +60,9 @@ /* Define to 1 if you have the header file. */ #undef HAVE_UNISTD_H +/* Hostbin executable */ +#undef HOSTBIN_PATH + /* Use keyfile for authentication */ #undef KEYFILE diff --git a/src/server/configure b/src/server/configure index d902d576..c9c52bda 100755 --- a/src/server/configure +++ b/src/server/configure @@ -959,6 +959,7 @@ enable_libtool_lock with_default_port with_localstorage enable_bluegene +with_hostbin enable_pipes enable_socket enable_shmem @@ -1656,6 +1657,7 @@ Optional Packages: --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-default-port=NUM TCP Port for Spindle server communication --with-localstorage=DIR Directory on back-ends for storing relocated files + --with-hostbin=EXE Executable for returning host lists for jobs --with-python-prefix=STR List of directories containing python installs --with-usage-logging=FILE @@ -5039,13 +5041,13 @@ if test "${lt_cv_nm_interface+set}" = set; then else lt_cv_nm_interface="BSD nm" echo "int some_variable = 0;" > conftest.$ac_ext - (eval echo "\"\$as_me:5042: $ac_compile\"" >&5) + (eval echo "\"\$as_me:5044: $ac_compile\"" >&5) (eval "$ac_compile" 2>conftest.err) cat conftest.err >&5 - (eval echo "\"\$as_me:5045: $NM \\\"conftest.$ac_objext\\\"\"" >&5) + (eval echo "\"\$as_me:5047: $NM \\\"conftest.$ac_objext\\\"\"" >&5) (eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out) cat conftest.err >&5 - (eval echo "\"\$as_me:5048: output\"" >&5) + (eval echo "\"\$as_me:5050: output\"" >&5) cat conftest.out >&5 if $GREP 'External.*some_variable' conftest.out > /dev/null; then lt_cv_nm_interface="MS dumpbin" @@ -6251,7 +6253,7 @@ ia64-*-hpux*) ;; *-*-irix6*) # Find out which ABI we are using. - echo '#line 6254 "configure"' > conftest.$ac_ext + echo '#line 6256 "configure"' > conftest.$ac_ext if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5 (eval $ac_compile) 2>&5 ac_status=$? @@ -8842,11 +8844,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:8845: $lt_compile\"" >&5) + (eval echo "\"\$as_me:8847: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:8849: \$? = $ac_status" >&5 + echo "$as_me:8851: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. @@ -9181,11 +9183,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:9184: $lt_compile\"" >&5) + (eval echo "\"\$as_me:9186: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:9188: \$? = $ac_status" >&5 + echo "$as_me:9190: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. @@ -9286,11 +9288,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:9289: $lt_compile\"" >&5) + (eval echo "\"\$as_me:9291: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:9293: \$? = $ac_status" >&5 + echo "$as_me:9295: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized @@ -9341,11 +9343,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:9344: $lt_compile\"" >&5) + (eval echo "\"\$as_me:9346: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:9348: \$? = $ac_status" >&5 + echo "$as_me:9350: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized @@ -12144,7 +12146,7 @@ else lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2 lt_status=$lt_dlunknown cat > conftest.$ac_ext <<_LT_EOF -#line 12147 "configure" +#line 12149 "configure" #include "confdefs.h" #if HAVE_DLFCN_H @@ -12240,7 +12242,7 @@ else lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2 lt_status=$lt_dlunknown cat > conftest.$ac_ext <<_LT_EOF -#line 12243 "configure" +#line 12245 "configure" #include "confdefs.h" #if HAVE_DLFCN_H @@ -14260,11 +14262,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:14263: $lt_compile\"" >&5) + (eval echo "\"\$as_me:14265: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:14267: \$? = $ac_status" >&5 + echo "$as_me:14269: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. @@ -14359,11 +14361,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:14362: $lt_compile\"" >&5) + (eval echo "\"\$as_me:14364: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:14366: \$? = $ac_status" >&5 + echo "$as_me:14368: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized @@ -14411,11 +14413,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:14414: $lt_compile\"" >&5) + (eval echo "\"\$as_me:14416: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:14418: \$? = $ac_status" >&5 + echo "$as_me:14420: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized @@ -15485,6 +15487,17 @@ if test "x$OS_BUILD" == "x"; then OS_BUILD=linux fi +#Startup type + +# Check whether --with-hostbin was given. +if test "${with_hostbin+set}" = set; then + withval=$with_hostbin; +cat >>confdefs.h <<_ACEOF +#define HOSTBIN_PATH "${withval}" +_ACEOF + +fi + #Runmode detection (pipe/socket or cobo/msocket communications) if test "x$OS_BUILD" == "xlinux"; then diff --git a/src/server/startup/Makefile.am b/src/server/startup/Makefile.am index 6eb20df0..516652f8 100644 --- a/src/server/startup/Makefile.am +++ b/src/server/startup/Makefile.am @@ -4,7 +4,7 @@ AM_CPPFLAGS = -I$(top_srcdir)/../logging spindle_be_LDFLAGS = -static spindle_be_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_srcdir)/comlib -I$(top_srcdir)/auditserver -I$(top_srcdir)/../client/beboot -I$(top_srcdir)/../include -I$(top_srcdir)/../utils -I$(top_srcdir)/../cobo -spindle_be_SOURCES = spindle_be_main.cc spindle_be_serial.cc spindle_be.cc $(top_srcdir)/../utils/parseloc.c $(top_srcdir)/../utils/keyfile.c +spindle_be_SOURCES = spindle_be_main.cc spindle_be_serial.cc spindle_be_hostbin.cc spindle_be.cc $(top_srcdir)/../utils/parseloc.c $(top_srcdir)/../utils/keyfile.c spindle_be_LDADD = $(top_builddir)/cache/libldcs_cache.la $(top_builddir)/logging/libspindlelogc.la spindle_be_LDADD += $(MUNGE_LIBS) $(GCRYPT_LIBS) diff --git a/src/server/startup/Makefile.in b/src/server/startup/Makefile.in index ac6365e0..fdcdb4c1 100644 --- a/src/server/startup/Makefile.in +++ b/src/server/startup/Makefile.in @@ -60,6 +60,7 @@ am__installdirs = "$(DESTDIR)$(pkglibexecdir)" PROGRAMS = $(pkglibexec_PROGRAMS) am_spindle_be_OBJECTS = spindle_be-spindle_be_main.$(OBJEXT) \ spindle_be-spindle_be_serial.$(OBJEXT) \ + spindle_be-spindle_be_hostbin.$(OBJEXT) \ spindle_be-spindle_be.$(OBJEXT) spindle_be-parseloc.$(OBJEXT) \ spindle_be-keyfile.$(OBJEXT) spindle_be_OBJECTS = $(am_spindle_be_OBJECTS) @@ -264,7 +265,7 @@ top_srcdir = @top_srcdir@ AM_CPPFLAGS = -I$(top_srcdir)/../logging spindle_be_LDFLAGS = -static $(am__append_1) spindle_be_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_srcdir)/comlib -I$(top_srcdir)/auditserver -I$(top_srcdir)/../client/beboot -I$(top_srcdir)/../include -I$(top_srcdir)/../utils -I$(top_srcdir)/../cobo -spindle_be_SOURCES = spindle_be_main.cc spindle_be_serial.cc spindle_be.cc $(top_srcdir)/../utils/parseloc.c $(top_srcdir)/../utils/keyfile.c +spindle_be_SOURCES = spindle_be_main.cc spindle_be_serial.cc spindle_be_hostbin.cc spindle_be.cc $(top_srcdir)/../utils/parseloc.c $(top_srcdir)/../utils/keyfile.c spindle_be_LDADD = $(top_builddir)/cache/libldcs_cache.la \ $(top_builddir)/logging/libspindlelogc.la $(MUNGE_LIBS) \ $(GCRYPT_LIBS) $(am__append_2) $(am__append_3) $(am__append_4) \ @@ -363,6 +364,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/spindle_be-keyfile.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/spindle_be-parseloc.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/spindle_be-spindle_be.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/spindle_be-spindle_be_hostbin.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/spindle_be-spindle_be_main.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/spindle_be-spindle_be_serial.Po@am__quote@ @@ -478,6 +480,22 @@ spindle_be-spindle_be_serial.obj: spindle_be_serial.cc @AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ @am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(spindle_be_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o spindle_be-spindle_be_serial.obj `if test -f 'spindle_be_serial.cc'; then $(CYGPATH_W) 'spindle_be_serial.cc'; else $(CYGPATH_W) '$(srcdir)/spindle_be_serial.cc'; fi` +spindle_be-spindle_be_hostbin.o: spindle_be_hostbin.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(spindle_be_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT spindle_be-spindle_be_hostbin.o -MD -MP -MF $(DEPDIR)/spindle_be-spindle_be_hostbin.Tpo -c -o spindle_be-spindle_be_hostbin.o `test -f 'spindle_be_hostbin.cc' || echo '$(srcdir)/'`spindle_be_hostbin.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/spindle_be-spindle_be_hostbin.Tpo $(DEPDIR)/spindle_be-spindle_be_hostbin.Po +@am__fastdepCXX_FALSE@ $(AM_V_CXX) @AM_BACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='spindle_be_hostbin.cc' object='spindle_be-spindle_be_hostbin.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(spindle_be_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o spindle_be-spindle_be_hostbin.o `test -f 'spindle_be_hostbin.cc' || echo '$(srcdir)/'`spindle_be_hostbin.cc + +spindle_be-spindle_be_hostbin.obj: spindle_be_hostbin.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(spindle_be_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT spindle_be-spindle_be_hostbin.obj -MD -MP -MF $(DEPDIR)/spindle_be-spindle_be_hostbin.Tpo -c -o spindle_be-spindle_be_hostbin.obj `if test -f 'spindle_be_hostbin.cc'; then $(CYGPATH_W) 'spindle_be_hostbin.cc'; else $(CYGPATH_W) '$(srcdir)/spindle_be_hostbin.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/spindle_be-spindle_be_hostbin.Tpo $(DEPDIR)/spindle_be-spindle_be_hostbin.Po +@am__fastdepCXX_FALSE@ $(AM_V_CXX) @AM_BACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='spindle_be_hostbin.cc' object='spindle_be-spindle_be_hostbin.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(spindle_be_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o spindle_be-spindle_be_hostbin.obj `if test -f 'spindle_be_hostbin.cc'; then $(CYGPATH_W) 'spindle_be_hostbin.cc'; else $(CYGPATH_W) '$(srcdir)/spindle_be_hostbin.cc'; fi` + spindle_be-spindle_be.o: spindle_be.cc @am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(spindle_be_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT spindle_be-spindle_be.o -MD -MP -MF $(DEPDIR)/spindle_be-spindle_be.Tpo -c -o spindle_be-spindle_be.o `test -f 'spindle_be.cc' || echo '$(srcdir)/'`spindle_be.cc @am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/spindle_be-spindle_be.Tpo $(DEPDIR)/spindle_be-spindle_be.Po diff --git a/src/server/startup/spindle_be.cc b/src/server/startup/spindle_be.cc index e1d8a8a5..e675611d 100644 --- a/src/server/startup/spindle_be.cc +++ b/src/server/startup/spindle_be.cc @@ -51,6 +51,7 @@ static int unpack_data(spindle_args_t *args, void *buffer, int buffer_size) unpack_param(args->opts, buf, pos); unpack_param(args->shared_secret, buf, pos); unpack_param(args->use_launcher, buf, pos); + unpack_param(args->startup_type, buf, pos); unpack_param(args->location, buf, pos); unpack_param(args->pythonprefix, buf, pos); unpack_param(args->preloadfile, buf, pos); diff --git a/src/server/startup/spindle_be_hostbin.cc b/src/server/startup/spindle_be_hostbin.cc new file mode 100644 index 00000000..85c7d453 --- /dev/null +++ b/src/server/startup/spindle_be_hostbin.cc @@ -0,0 +1,23 @@ +/* +This file is part of Spindle. For copyright information see the COPYRIGHT +file in the top level directory, or at +https://github.com/hpc/Spindle/blob/master/COPYRIGHT + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License (as published by the Free Software +Foundation) version 2.1 dated February 1999. This program is distributed in the +hope that it will be useful, but WITHOUT ANY WARRANTY; without even the IMPLIED +WARRANTY OF MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms +and conditions of the GNU General Public License for more details. You should +have received a copy of the GNU Lesser General Public License along with this +program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA +*/ + +#include "spindle_launch.h" +#include "spindle_debug.h" + +int startHostbinBE(unsigned int port, unsigned int shared_secret) +{ + return spindleRunBE(port, shared_secret, NULL); +} diff --git a/src/server/startup/spindle_be_main.cc b/src/server/startup/spindle_be_main.cc index 98091940..ad108b95 100644 --- a/src/server/startup/spindle_be_main.cc +++ b/src/server/startup/spindle_be_main.cc @@ -32,15 +32,19 @@ static void initSecurity(); #if defined(HAVE_LMON) extern int startLaunchmonBE(int argc, char *argv[]); #endif - +extern int startHostbinBE(unsigned int port, unsigned int shared_secret); extern int startSerialBE(int argc, char *argv[]); + enum startup_type_t { lmon, - serial + serial, + hostbin }; startup_type_t startup_type; static int security_type; static int number; +static int port; +static int shared_secret; int main(int argc, char *argv[]) { @@ -73,6 +77,9 @@ int main(int argc, char *argv[]) case serial: result = startSerialBE(argc, argv); break; + case hostbin: + result = startHostbinBE(port, shared_secret); + break; default: err_printf("Unknown startup mode\n"); result = -1; @@ -111,16 +118,24 @@ static int parseCommandLine(int argc, char *argv[]) startup_type = serial; break; } + else if (strcmp(argv[i], "--spindle_hostbin") == 0) { + startup_type = hostbin; + break; + } } - i++; - if (i >= argc) return -1; + if (++i >= argc) return -1; security_type = atoi(argv[i]); - i++; - if (i >= argc) return -1; + if (++i >= argc) return -1; number = atoi(argv[i]); + if (startup_type == hostbin) { + if (++i >= argc) return -1; + port = atoi(argv[i]); + number = atoi(argv[i]); + } + return 0; }