Merge pull request #18 from hpc/features/readlink

Improved support for fstat, readlink, and error returns

.gitignore

@@ -4,6 +4,8 @@
 /src/client/autom4te.cache
 /src/fe/autom4te.cache
 
+tags
+
 #Need if someone does an in-source configure/build
 Makefile
 config.h

config.h.in

@@ -103,6 +103,9 @@
 /* Define to 1 if you have the ANSI C header files. */
 #undef STDC_HEADERS
 
+/* Default recource manager */
+#undef TESTRM
+
 /* If the compiler supports a TLS storage class define it to that here */
 #undef TLS
 

configure

@@ -653,7 +653,6 @@ MPI_CFLAGS
 MPICC
 BLD_TESTSUITE_FALSE
 BLD_TESTSUITE_TRUE
-TESTRM
 AM_CFLAGS
 LMON_DYNAMIC_FALSE
 LMON_DYNAMIC_TRUE
@@ -686,6 +685,7 @@ SOCKETS_FALSE
 SOCKETS_TRUE
 PIPES_FALSE
 PIPES_TRUE
+TESTRM
 CXXCPP
 CPP
 OTOOL64
@@ -824,6 +824,8 @@ enable_libtool_lock
 with_default_port
 with_default_num_ports
 with_localstorage
+with_testrm
+with_rm
 enable_bluegene
 with_hostbin
 enable_pipes
@@ -845,7 +847,6 @@ with_launchmon
 with_launchmon_incdir
 with_launchmon_libdir
 with_launchmon_rmcommdir
-with_testrm
 enable_testsuite
 '
       ac_precious_vars='build_alias
@@ -1533,6 +1534,9 @@ Optional Packages:
                           Number of TCP/IP ports to scan for Spindle server
                           communication
   --with-localstorage=DIR Directory on back-ends for storing relocated files
+  --with-testrm=ARM
+  --with-rm=ARG           Default resource manager (valid options are 'slurm'
+                          and 'flux')
   --with-hostbin=EXE      Executable for returning host lists for jobs
   --with-python-prefix=STR
                           List of directories containing python installs
@@ -1548,8 +1552,6 @@ Optional Packages:
                           Launchmon library directory
   --with-launchmon-rmcommdir=DIR
                           RM communication library LaunchMON uses
-  --with-testrm=ARG       Resource manager to use for tests (valid options are
-                          'slurm' and 'flux')
 
 Some influential environment variables:
   CC          C compiler command
@@ -16244,6 +16246,32 @@ _ACEOF
 
 
 
+TESTRM=unknown
+
+# Check whether --with-testrm was given.
+if test "${with_testrm+set}" = set; then :
+  withval=$with_testrm; TESTRM=${withval}
+fi
+
+
+# Check whether --with-rm was given.
+if test "${with_rm+set}" = set; then :
+  withval=$with_rm; TESTRM=${withval}
+fi
+
+if test "x$TESTRM" == "xunknown"; then
+   { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: \"--with-rm not specified.  Tests will not work\"" >&5
+$as_echo "$as_me: WARNING: \"--with-rm not specified.  Tests will not work\"" >&2;}
+fi
+
+
+cat >>confdefs.h <<_ACEOF
+#define TESTRM "$TESTRM"
+_ACEOF
+
+
+
+
 #OS Detection
 # Check whether --enable-bluegene was given.
 if test "${enable_bluegene+set}" = set; then :
@@ -16890,17 +16918,6 @@ fi
 AM_CFLAGS=-Wall
 
 
-
-# Check whether --with-testrm was given.
-if test "${with_testrm+set}" = set; then :
-  withval=$with_testrm; TESTRM=${withval}
-else
-  TESTRM=unknown; { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: --with-testrm not specified. Spindle tests will not work" >&5
-$as_echo "$as_me: WARNING: --with-testrm not specified. Spindle tests will not work" >&2;}
-fi
-
-
-
 #MPI detection
 # Check whether --enable-testsuite was given.
 if test "${enable_testsuite+set}" = set; then :

configure.ac

@@ -23,12 +23,6 @@ AX_TLS
 #Default flags
 AC_SUBST([AM_CFLAGS], [-Wall])
 
-AC_ARG_WITH(testrm,
-            [AS_HELP_STRING([--with-testrm=ARG],[Resource manager to use for tests (valid options are 'slurm' and 'flux')])],
-            [TESTRM=${withval}],
-            [TESTRM=unknown; AC_MSG_WARN([--with-testrm not specified. Spindle tests will not work])])
-AC_SUBST(TESTRM)
-
 #MPI detection
 AC_ARG_ENABLE(testsuite,
               [AS_HELP_STRING([--enable-testsuite],[Chooses whether to build the Spindle testsuite. Default yes])],

configure.common.ac

@@ -22,6 +22,23 @@ AC_DEFINE_UNQUOTED([SPINDLE_MAX_PORT],[$(($SPINDLE_PORT + $NUM_COBO_PORTS - 1))]
 AC_DEFINE_UNQUOTED([SPINDLE_LOC],"[$SPINDLE_LOC]",[The default local directory for Spindle])
 
 
+TESTRM=unknown
+AC_ARG_WITH(testrm,
+       [AS_HELP_STRING([--with-testrm=ARM],[])],
+       [TESTRM=${withval}],
+       [])
+AC_ARG_WITH(rm,
+        [AS_HELP_STRING([--with-rm=ARG],[Default resource manager (valid options are 'slurm' and 'flux')])],
+        [TESTRM=${withval}],
+        [])
+if test "x$TESTRM" == "xunknown"; then
+   AC_WARN(["--with-rm not specified.  Tests will not work"])
+fi
+AC_SUBST(TESTRM)
+AC_DEFINE_UNQUOTED([TESTRM], "[$TESTRM]", [Default recource manager])
+
+
+
 #OS Detection
 AC_ARG_ENABLE(bluegene,
               [AS_HELP_STRING([--enable-bluegene],[Build Spindle for BlueGene/Q])],
@@ -217,7 +234,7 @@ AC_DEFUN([CHOOSE_SEC],[
         LIBS=$LIBS_HOLD
 
         #Check for libgcrypt availability
-        GCRYPT_LIBS="$GCRYPT_LIBS -lgcrypt"
+        GCRYPT_LIBS="$GCRYPT_LIBS -lgcrypt -lgpg-error"
         CFLAGS="$CFLAGS $GCRYPT_CFLAGS"
         LIBS="$LIBS $GCRYPT_LIBS"
         HAVE_GCRYPT="true"

doc/spindle.1

@@ -19,6 +19,18 @@ spindle -q --reloc-python=no mpirun -np 512 ./my_mpi_app apparg1 apparg2
 .fi
 .RE
 
+Spindle can also run in a persistent session mode, where Spindle can provide scalable loading to ensemble application runs (running many small jobs rather than one big job).  In this case spindle may be started as something like:
+
+.nf
+.RS
+export ID=`spindle --start-session`
+spindle --run-in-session $ID srun -n 4 /home/me/my_mpi_app &
+spindle --run-in-session $ID srun -n 4 /home/me/my_mpi_app &
+spindle --run-in-session $ID srun -n 4 /home/me/my_mpi_app &
+spindle --run-in-session $ID srun -n 4 /home/me/my_mpi_app &
+wait
+spindle --end-session $ID
+
 .SH OPTIONS
 .TP
 \fB\-a \fIyes\fR|\fIno\fR, \fB\-\-reloc\-aout\fR=\fIyes\fR|\fIno\fR
@@ -72,6 +84,18 @@ Use gcrypt based signatures to authenticate connections between Spindle servers,
 \fB\-\-security\-none\fR
 Spindle will not attempt to authenticate connections between Spindle server.  Any outside users may be able to connect into Spindle's server network and provide/request files as the user.  This option must be explicitly enabled at Spindle build time before it will be available.
 
+.TP
+\fB\-\-start\-session\fR
+Spindle will spawn a persistent session daemon, print an alpha-numeric session ID to stdout, and exit.  Spindle's other session arguments take the ID and can be used to run multiple jobs in the same spindle session.  Each job will share the same file caches and can run concurrently.  Any other spindle arguments (security, cache location, etc) should be specified on the command line when starting the spindle session.
+
+.TP
+\fB\-\-run\-in\-session\fR \fISESSION_ID\fR
+Run a new spindle job in an existing session.  The required SESSION_ID argument is the alpha-numeric session ID printed by \fI\-\-start-session\fR.  Any other arguments on the command line, besides the job launch command, will be ignored.  Instead the job will run in the configuration specified when the session was started.
+
+.TP
+\fB\-\-end\-session\fR \fISESSION_ID\fR
+End a spindle session and clean up related caches.  The required SESSION_ID argument is the alpha-numeric session ID printed by \fI\-\-start-session\fR.  This option should not be used while jobs are still running in the session.
+
 .TP
 \fB\-\-no\-mpi\fR
 Tells spindle to run a serial job rather than an MPI job.  Spindle does not provide significant performance benefits for serial jobs, but this option can be useful for debugging.
@@ -114,7 +138,8 @@ Spindle requires local storage on each node (such as a ramdisk or SSD) for stori
 
 .TP
 \fB\-r\fR \fIPATH\fR, \fB\-\-python\-prefix=\fIPATH\fR
-Spindle can provide a better quality-of-service on Python programs if it knows the prefix where Python was installed.  This option provides a colon-separated list of directories where Spindle may find Python installations.  The directories in \fIPATH\fR are treated as prefixes, and any file operation in their subdirectories are assumed to be on Python files.  This directory list should not contain any directories where the application will make writes (so it would be a bad idea to add '/' to this list).
+\fB\-r\fR \fIPATH\fR, \fB\-\-cache\-prefix=\fIPATH\fR
+Spindle can provide a better quality-of-service on Python and other interpreted programs if it knows the prefix where the interpreter stores libraries.  This option provides a colon-separated list of directories where Spindle may find interpreter libraries.  The directories in \fIPATH\fR are treated as prefixes, and any file read operation in their subdirectories will be scalably broadcast through spindle.  This directory list should not contain any directories where the application will make writes (so it would be a bad idea to add '/' to this list).  The \fI\-\-cache-prefix\fR and \fI\-\-python-prefix\fR options are aliases.
 
 .TP
 \fB\-s\fR \fIyes\fR|\fIno\fR, \fB\-\-strip=\fIyes\fR|\fIno\fR
@@ -147,11 +172,11 @@ Setting the \fBSPINDLE_DEBUG\fR environment variable before running Spindle will
 Spindle will return after the completion of the MPI launcher and with its error code.  If Spindle encounters a fatal error independent of the MPI launcher it will return with a non-zero exit code.
 
 .SH COPYRIGHT
-Copyright (c) 2014, Lawrence Livermore National Security, LLC. Produced at
+Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
 the Lawrence Livermore National Laboratory. Written by Matthew LeGendre 
 legendre1@llnl.gov. CODE-636292. All rights reserved.
 
-Copyright (c) 2014, Juelich Supercomputing Center.  Written by Wolfgang Frings
+Copyright (c) 2017, Juelich Supercomputing Center.  Written by Wolfgang Frings
 W.Frings@fz-juelich.de. All rights reserved.
 
 License: LGPL 2.1

doc/spindle_launch_README.md

@@ -106,6 +106,8 @@ we have to break ABI compatibility.
             should be a OPT_SEC_* value.
         -   `OPT_GET_SEC(OPT)` - Returns the OPT_SEC part of the opt_t
             bitfield specified by OPT.
+   -    'OPT_SESSION' - Run in session mode, where spindle caches persist
+        between runs.
 
 
 - `typedef struct { ... } spindle_args_t`

src/client/Makefile.in

@@ -270,6 +270,7 @@ SED = @SED@
 SET_MAKE = @SET_MAKE@
 SHELL = @SHELL@
 STRIP = @STRIP@
+TESTRM = @TESTRM@
 VERSION = @VERSION@
 abs_builddir = @abs_builddir@
 abs_srcdir = @abs_srcdir@

src/client/auditclient/Makefile.in

@@ -308,6 +308,7 @@ SED = @SED@
 SET_MAKE = @SET_MAKE@
 SHELL = @SHELL@
 STRIP = @STRIP@
+TESTRM = @TESTRM@
 VERSION = @VERSION@
 abs_builddir = @abs_builddir@
 abs_srcdir = @abs_srcdir@

src/client/beboot/Makefile.in

@@ -242,6 +242,7 @@ SED = @SED@
 SET_MAKE = @SET_MAKE@
 SHELL = @SHELL@
 STRIP = @STRIP@
+TESTRM = @TESTRM@
 VERSION = @VERSION@
 abs_builddir = @abs_builddir@
 abs_srcdir = @abs_srcdir@

src/client/beboot/spindle_bootstrap.c

@@ -218,13 +218,14 @@ static void launch_daemon(char *location)
 
 static void get_executable()
 {
+   int errcode = 0;
    if (!(opts & OPT_RELOCAOUT)) {
       executable = *cmdline;
       return;
    }
 
    debug_printf2("Sending request for executable %s\n", *cmdline);
-   exec_pathsearch(ldcsid, *cmdline, &executable);
+   exec_pathsearch(ldcsid, *cmdline, &executable, &errcode);
 
    if (executable == NULL) {
       executable = *cmdline;
@@ -260,13 +261,14 @@ static void adjust_script()
 static void get_clientlib()
 {
    char *default_libstr = (opts & OPT_SUBAUDIT) ? default_subaudit_libstr : default_audit_libstr;
-
+   int errorcode;
+
    if (!(opts & OPT_RELOCAOUT)) {
       client_lib = default_libstr;
       return;
    }
 
-   send_file_query(ldcsid, default_libstr, &client_lib);
+   send_file_query(ldcsid, default_libstr, &client_lib, &errorcode);
    if (client_lib == NULL) {
       client_lib = default_libstr;
       err_printf("Failed to relocate client library %s\n", default_libstr);
@@ -277,9 +279,41 @@ static void get_clientlib()
    }
 }
 
-int get_relocated_file(int fd, const char *name, char** newname)
+extern int read_buffer(char *localname, char *buffer, int size);
+int get_stat_result(int fd, const char *path, int is_lstat, int *exists, struct stat *buf)
+{
+   int result;
+   char buffer[MAX_PATH_LEN+1];
+   char *newpath;
+   int found_file = 0;
+
+   if (!found_file) {
+      result = send_stat_request(fd, (char *) path, is_lstat, buffer);
+      if (result == -1) {
+         *exists = 0;
+         return -1;
+      }
+      newpath = buffer[0] != '\0' ? buffer : NULL;
+   }
+
+   if (newpath == NULL) {
+      *exists = 0;
+      return 0;
+   }
+   *exists = 1;
+
+   result = read_buffer(newpath, (char *) buf, sizeof(*buf));
+   if (result == -1) {
+      err_printf("Failed to read stat info for %s from %s\n", path, newpath);
+      *exists = 0;
+      return -1;
+   }
+   return 0;
+}
+
+int get_relocated_file(int fd, const char *name, char** newname, int *errcode)
 {
-   return send_file_query(fd, (char *) name, newname);
+   return send_file_query(fd, (char *) name, newname, errcode);
 }
 
 /**
@@ -386,7 +420,7 @@ int main(int argc, char *argv[])
     * Exec the user's application.
     **/
    setup_environment();
-   execv(executable, cmdline);
+   execvp(executable, cmdline);
 
    /**
     * Exec error handling

src/client/biter/Makefile.in

@@ -239,6 +239,7 @@ SED = @SED@
 SET_MAKE = @SET_MAKE@
 SHELL = @SHELL@
 STRIP = @STRIP@
+TESTRM = @TESTRM@
 VERSION = @VERSION@
 abs_builddir = @abs_builddir@
 abs_srcdir = @abs_srcdir@

src/client/client/Makefile.am

@@ -14,7 +14,7 @@ AM_CFLAGS = -fvisibility=hidden
 
 AM_CPPFLAGS = -I$(top_srcdir)/../logging -I$(top_srcdir)/client_comlib -I$(top_srcdir)/../include -I$(top_srcdir)/shm_cache
 
-INTERCEPT_SRCS = intercept_open.c intercept_exec.c intercept_stat.c intercept_spindleapi.c intercept.c
+INTERCEPT_SRCS = intercept_open.c intercept_exec.c intercept_stat.c intercept_readlink.c intercept_spindleapi.c intercept.c
 
 BASE_SRCS = client.c should_intercept.c exec_util.c remap_exec.c rogot.c $(top_srcdir)/../utils/parseloc.c