This repository has been archived by the owner. It is now read-only.
Permalink
Browse files

Teach IOR about GPFS hints (gpfs_fcntl)

GPFS supports a "gpfs_fcntl" method for hinting various things,
including "i'm about to write this block of data".  Let's see if, for
the cost of a few system calls, we can wrangle the GPFS locking system
into allowing concurrent access with less overhead. (new IOR parameter
gpfsHintAccess)

Also, drop all locks on a file immediately after open/creation in the
shared file case, since we know all processes will touch unique regions
of the file.  It may or may not be a good idea to release all file locks
after opening.  Processes will then have to re-acquire locks already
held.   (new IOR parameter gpfsReleaseToken)
  • Loading branch information...
roblatham00 authored and morrone committed Sep 26, 2013
1 parent 6a7ddfe commit aa604c1d38de803aa0db3f3abb5515e0ed1857ea
Showing with 131 additions and 1 deletion.
  1. +1 −1 configure.ac
  2. +7 −0 doc/USER_GUIDE
  3. +104 −0 src/aiori-POSIX.c
  4. +4 −0 src/ior.c
  5. +5 −0 src/ior.h
  6. +10 −0 src/parse_options.c
View
@@ -20,7 +20,7 @@ AX_PROG_CC_MPI
# Checks for libraries.
# Checks for header files.
-AC_CHECK_HEADERS([fcntl.h libintl.h stdlib.h string.h strings.h sys/ioctl.h sys/param.h sys/statfs.h sys/statvfs.h sys/time.h unistd.h wchar.h])
+AC_CHECK_HEADERS([fcntl.h libintl.h stdlib.h string.h strings.h sys/ioctl.h sys/param.h sys/statfs.h sys/statvfs.h sys/time.h unistd.h wchar.h gpfs.h gpfs_fcntl.h])
# Checks for typedefs, structures, and compiler characteristics.
AC_TYPE_SIZE_T
View
@@ -341,6 +341,13 @@ LUSTRE-SPECIFIC:
* lustreIgnoreLocks - disable lustre range locking [0]
+GPFS-SPECIFIC:
+================
+ * gpfsHintAccess - use gpfs_fcntl hints to pre-declare accesses
+
+ * gpfsReleaseToken - immediately after opening or creating file, release
+ all locks. Might help mitigate lock-revocation
+ traffic when many proceses write/read to same file.
***********************
* 5. VERBOSITY LEVELS *
View
@@ -32,6 +32,13 @@
#include <lustre/lustre_user.h>
#endif
+#ifdef HAVE_GPFS_H
+#include <gpfs.h>
+#endif
+#ifdef HAVE_GPFS_FCNTL_H
+#include <gpfs_fcntl.h>
+#endif
+
#include "ior.h"
#include "aiori.h"
#include "iordef.h"
@@ -91,6 +98,78 @@ void set_o_direct_flag(int *fd)
*fd |= O_DIRECT;
}
+#ifdef HAVE_GPFS_FCNTL_H
+void gpfs_free_all_locks(int fd)
+{
+ int rc;
+ struct {
+ gpfsFcntlHeader_t header;
+ gpfsFreeRange_t release;
+ } release_all;
+ release_all.header.totalLength = sizeof(release_all);
+ release_all.header.fcntlVersion = GPFS_FCNTL_CURRENT_VERSION;
+ release_all.header.fcntlReserved = 0;
+
+ release_all.release.structLen = sizeof(release_all.release);
+ release_all.release.structType = GPFS_FREE_RANGE;
+ release_all.release.start = 0;
+ release_all.release.length = 0;
+
+ rc = gpfs_fcntl(fd, &release_all);
+ if (verbose >= VERBOSE_0 && rc != 0) {
+ EWARN("gpfs_fcntl release all locks hint failed.");
+ }
+}
+void gpfs_access_start(int fd, IOR_offset_t length, IOR_param_t *param, int access)
+{
+ int rc;
+ struct {
+ gpfsFcntlHeader_t header;
+ gpfsAccessRange_t access;
+ } take_locks;
+
+ take_locks.header.totalLength = sizeof(take_locks);
+ take_locks.header.fcntlVersion = GPFS_FCNTL_CURRENT_VERSION;
+ take_locks.header.fcntlReserved = 0;
+
+ take_locks.access.structLen = sizeof(take_locks.access);
+ take_locks.access.structType = GPFS_ACCESS_RANGE;
+ take_locks.access.start = param->offset;
+ take_locks.access.length = length;
+ take_locks.access.isWrite = (access == WRITE);
+
+ rc = gpfs_fcntl(fd, &take_locks);
+ if (verbose >= VERBOSE_2 && rc != 0) {
+ EWARN("gpfs_fcntl access range hint failed.");
+ }
+}
+
+void gpfs_access_end(int fd, IOR_offset_t length, IOR_param_t *param, int access)
+{
+ int rc;
+ struct {
+ gpfsFcntlHeader_t header;
+ gpfsFreeRange_t free;
+ } free_locks;
+
+
+ free_locks.header.totalLength = sizeof(free_locks);
+ free_locks.header.fcntlVersion = GPFS_FCNTL_CURRENT_VERSION;
+ free_locks.header.fcntlReserved = 0;
+
+ free_locks.free.structLen = sizeof(free_locks.free);
+ free_locks.free.structType = GPFS_FREE_RANGE;
+ free_locks.free.start = param->offset;
+ free_locks.free.length = length;
+
+ rc = gpfs_fcntl(fd, &free_locks);
+ if (verbose >= VERBOSE_2 && rc != 0) {
+ EWARN("gpfs_fcntl free range hint failed.");
+ }
+}
+
+#endif
+
/*
* Creat and open a file through the POSIX interface.
*/
@@ -166,6 +245,14 @@ static void *POSIX_Create(char *testFileName, IOR_param_t * param)
}
#endif /* HAVE_LUSTRE_LUSTRE_USER_H */
+#ifdef HAVE_GPFS_FCNTL_H
+ /* in the single shared file case, immediately release all locks, with
+ * the intent that we can avoid some byte range lock revocation:
+ * everyone will be writing/reading from individual regions */
+ if (param->gpfs_release_token ) {
+ gpfs_free_all_locks(*fd);
+ }
+#endif
return ((void *)fd);
}
@@ -201,6 +288,11 @@ static void *POSIX_Open(char *testFileName, IOR_param_t * param)
}
#endif /* HAVE_LUSTRE_LUSTRE_USER_H */
+#ifdef HAVE_GPFS_FCNTL_H
+ if(param->gpfs_release_token) {
+ gpfs_free_all_locks(*fd);
+ }
+#endif
return ((void *)fd);
}
@@ -218,6 +310,13 @@ static IOR_offset_t POSIX_Xfer(int access, void *file, IOR_size_t * buffer,
fd = *(int *)file;
+#ifdef HAVE_GPFS_FCNTL_H
+ if (param->gpfs_hint_access) {
+ gpfs_access_start(fd, length, param, access);
+ }
+#endif
+
+
/* seek to offset */
if (lseek64(fd, param->offset, SEEK_SET) == -1)
ERR("lseek64() failed");
@@ -268,6 +367,11 @@ static IOR_offset_t POSIX_Xfer(int access, void *file, IOR_size_t * buffer,
ptr += rc;
xferRetries++;
}
+#ifdef HAVE_GPFS_FCNTL_H
+ if (param->gpfs_hint_access) {
+ gpfs_access_end(fd, length, param, access);
+ }
+#endif
return (length);
}
View
@@ -1601,6 +1601,10 @@ static void ShowTest(IOR_param_t * test)
test->setTimeStampSignature);
fprintf(stdout, "\t%s=%d\n", "collective", test->collective);
fprintf(stdout, "\t%s=%lld", "segmentCount", test->segmentCount);
+#ifdef HAVE_GPFS_FCNTL_H
+ fprintf(stdout, "\t%s=%d\n", "gpfsHintAccess", test->gpfs_hint_access);
+ fprintf(stdout, "\t%s=%d\n", "gpfsReleaseToken", test->gpfs_release_token);
+#endif
if (strcmp(test->api, "HDF5") == 0) {
fprintf(stdout, " (datasets)");
}
View
@@ -126,6 +126,11 @@ typedef struct
int lustre_set_striping; /* flag that we need to set lustre striping */
int lustre_ignore_locks;
+ /* gpfs variables */
+ int gpfs_hint_access; /* use gpfs "access range" hint */
+ int gpfs_release_token; /* immediately release GPFS tokens after
+ creating or opening a file */
+
int id; /* test's unique ID */
int intraTestBarriers; /* barriers between open/op and op/close */
} IOR_param_t;
View
@@ -263,6 +263,16 @@ void DecodeDirective(char *line, IOR_param_t *params)
ERR("ior was not compiled with Lustre support");
#endif
params->lustre_ignore_locks = atoi(value);
+ } else if (strcasecmp(option, "gpfshintaccess") == 0) {
+#ifndef HAVE_GPFS_FCNTL_H
+ ERR("ior was not compiled with GPFS hint support");
+#endif
+ params->gpfs_hint_access = atoi(value);
+ } else if (strcasecmp(option, "gpfsreleasetoken") == 0) {
+#ifndef HAVE_GPFS_FCNTL_H
+ ERR("ior was not compiled with GPFS hint support");
+#endif
+ params->gpfs_release_token = atoi(value);
} else if (strcasecmp(option, "numtasks") == 0) {
params->numTasks = atoi(value);
RecalculateExpectedFileSize(params);

0 comments on commit aa604c1

Please sign in to comment.