Throttle implementation

packaging/rhel/xrootd.spec.in

@@ -628,6 +628,7 @@ fi
 %{_libdir}/libXrdHttp-4.so
 %{_libdir}/libXrdOssSIgpfsT-4.so
 %{_libdir}/libXrdServer.so.*
+%{_libdir}/libXrdThrottle-4.so
 
 %files server-devel
 %defattr(-,root,root,-)

src/XrdPlugins.cmake

@@ -8,6 +8,7 @@ set( LIB_XRD_BWM        XrdBwm-${PLUGIN_VERSION} )
 set( LIB_XRD_PSS        XrdPss-${PLUGIN_VERSION} )
 set( LIB_XRD_GPFS       XrdOssSIgpfsT-${PLUGIN_VERSION} )
 set( LIB_XRD_ZCRC32     XrdCksCalczcrc32-${PLUGIN_VERSION} )
+set( LIB_XRD_THROTTLE   XrdThrottle-${PLUGIN_VERSION} )
 
 #-------------------------------------------------------------------------------
 # Shared library version
@@ -100,9 +101,34 @@ set_target_properties(
   INTERFACE_LINK_LIBRARIES ""
   LINK_INTERFACE_LIBRARIES "" )
 
+#-------------------------------------------------------------------------------
+# The XrdThrottle lib
+#-------------------------------------------------------------------------------
+add_library(
+  ${LIB_XRD_THROTTLE}
+  SHARED
+  XrdOfs/XrdOfsFS.cc
+  XrdThrottle/XrdThrottle.hh           XrdThrottle/XrdThrottleTrace.hh
+  XrdThrottle/XrdThrottleFileSystem.cc
+  XrdThrottle/XrdThrottleFileSystemConfig.cc
+  XrdThrottle/XrdThrottleFile.cc
+  XrdThrottle/XrdThrottleManager.cc    XrdThrottle/XrdThrottleManager.hh
+)
+
+target_link_libraries(
+  ${LIB_XRD_THROTTLE}
+  XrdServer
+  XrdUtils )
+
+set_target_properties(
+  ${LIB_XRD_THROTTLE}
+  PROPERTIES
+  INTERFACE_LINK_LIBRARIES ""
+  LINK_INTERFACE_LIBRARIES "" )
+
 #-------------------------------------------------------------------------------
 # Install
 #-------------------------------------------------------------------------------
 install(
-  TARGETS ${LIB_XRD_PSS} ${LIB_XRD_BWM} ${LIB_XRD_GPFS} ${LIB_XRD_ZCRC32}
+  TARGETS ${LIB_XRD_PSS} ${LIB_XRD_BWM} ${LIB_XRD_GPFS} ${LIB_XRD_ZCRC32} ${LIB_XRD_THROTTLE}
   LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} )

src/XrdThrottle/README

@@ -0,0 +1,77 @@
+
+INTRODUCTION
+
+This "file system" implementation for Xrootd throttles request rates
+passed through to the real underlying filesystem layer.  It has two
+goals:
+
+- Prevent users from overloading a filesystem through Xrootd.
+- Provide a level of fairness between different users.
+
+Here, "fairness" is loosely done - while it is done across all open
+file handles for a given user, it allows them to opportunistically
+utilize bandwidth allocated to, but not used by, others.  There's no
+concept of fairshare or history beyond the previous time interval (by default,
+1 second).  Fairness is enforced by trying to delaying IO the same
+amount *per user*, regardless of how many open file handles there are.
+
+When loaded, in order for the plugin to perform timings for IO, asynchronous
+requests are handled synchronously and mmap-based reads are disabled.  It is
+believed this impact is minimal.
+
+Once a throttle limit is hit, the plugin will start delaying the start of
+new IO requests until the server is back below the throttle.  The granularity
+of measurement, by default, is a 1 second interval; the overall amount of delay
+may be minimal if the server is only slightly above limits.
+
+USAGE
+
+To load the plugin, add it as the first xrootd.fslib in the configuration file:
+
+xrootd.fslib throttle default
+
+Unless limits are explicitly set, the plugin will only record (and, if configured,
+log) usage statistics.
+
+To set a throttle, add a line as follows:
+
+throttle.throttle [concurrency CONCUR] [data RATE]
+
+The two options are:
+
+  - CONCUR: Set the level of IO concurrency allowed.  This works in a similar
+    manner to system load in Linux; we sum up the total amount of time spent
+    waiting on all IO requests per second.  So, if there are two simultaneous
+    requests, each of which take 1 second to service, we have a concurrency of
+    2.  If we have 100 simultaneous IO requests, each of which is services in
+    1 millisecond, then the IO load is 0.1.
+  - RATE: Limit for the total data rate (MB/s) from the underlying filesystem.
+    This number is measured in bytes.
+
+NOTES:
+- The throttles are applied to the aggregate of reads and writes; they are not
+  considered seperately.
+- In almost all cases, service administrators will want to set concurrency, NOT,
+  data.  Concurrency measures how much work is being done by the filesystem;
+  data rate is weak indicator of filesystem activity due to the effects of
+  the page cache.  We do not offer the option to limit number of clients or
+  IOPS in this plugin for similar reasons: neither metric strongly corresponds
+  to filesystem activity (due to idle clients or IO requests serviced from cache).
+- As sites commonly run multiple servers, setting the data rate throttle is
+  not useful to limit wide area network traffic.  If you want to limit network
+  activity, investigate QoS on the site's network router.  Worst case, configuring
+  the host-level network queueing will be more CPU-efficient than setting the
+  data rates from within Xrootd.  The sole advantage of throttling data rates
+  from within Xrootd is being able to provide fairness across users.
+
+To log throttle-related activity, set:
+
+throttle.trace [all] [off|none] [bandwidth] [ioload] [debug]
+
+- all: All debugging statements are enabled.
+- off, none: No debugging statements are enabled.
+- bandwidth: Log bandwidth-usage-related statistics.
+- ioload: Log concurrency-related statistics.
+- debug: Log all throttle-related information; this is very chatty and aims
+  to provide developers with enough information to debug the throttle's activity.
+

src/XrdThrottle/XrdThrottle.hh

@@ -0,0 +1,232 @@
+#ifndef __XRDTHROTTLE_H_
+#define __XRDTHROTTLE_H_
+
+#include <memory>
+#include <string>
+
+#include "XrdVersion.hh"
+#include "XrdSys/XrdSysError.hh"
+#include "XrdSfs/XrdSfsInterface.hh"
+
+#include "XrdThrottle/XrdThrottleTrace.hh"
+#include "XrdThrottle/XrdThrottleManager.hh"
+
+class XrdSysLogger;
+class XrdOucStream;
+
+namespace XrdThrottle {
+
+class FileSystem;
+
+class File : public XrdSfsFile {
+
+friend class FileSystem;
+
+public:
+
+   virtual int
+   open(const char                *fileName,
+              XrdSfsFileOpenMode   openMode,
+              mode_t               createMode,
+        const XrdSecEntity        *client,
+        const char                *opaque = 0);
+
+   virtual int
+   close();
+
+   virtual int
+   fctl(const int               cmd,
+        const char             *args,
+              XrdOucErrInfo    &out_error);
+
+   virtual const char *
+   FName();
+
+   virtual int
+   getMmap(void **Addr, off_t &Size);
+
+   virtual int
+   read(XrdSfsFileOffset   fileOffset,   // Preread only
+        XrdSfsXferSize     amount);
+
+   virtual XrdSfsXferSize
+   read(XrdSfsFileOffset   fileOffset,
+        char              *buffer,
+        XrdSfsXferSize     buffer_size);
+
+   virtual int
+   read(XrdSfsAio *aioparm);
+
+   virtual XrdSfsXferSize
+   write(XrdSfsFileOffset   fileOffset,
+         const char        *buffer,
+         XrdSfsXferSize     buffer_size);
+
+   virtual int
+   write(XrdSfsAio *aioparm);
+
+   virtual int
+   sync();
+
+   virtual int
+   sync(XrdSfsAio *aiop);
+
+   virtual int
+   stat(struct stat *buf);
+
+   virtual int
+   truncate(XrdSfsFileOffset   fileOffset);
+
+   virtual int
+   getCXinfo(char cxtype[4], int &cxrsz);
+
+   virtual int
+   SendData(XrdSfsDio         *sfDio,
+            XrdSfsFileOffset   offset,
+            XrdSfsXferSize     size);
+
+private:
+   File(const char *user, int monid, std::auto_ptr<XrdSfsFile>, XrdThrottleManager &throttle, XrdSysError &eroute);
+
+   virtual
+   ~File();
+
+   std::auto_ptr<XrdSfsFile> m_sfs;
+   int m_uid; // A unique identifier for this user; has no meaning except for the fairshare.
+   std::string m_loadshed;
+   std::string m_user;
+   XrdThrottleManager &m_throttle;
+   XrdSysError &m_eroute;
+};
+
+class FileSystem : public XrdSfsFileSystem
+{
+
+friend XrdSfsFileSystem * XrdSfsGetFileSystem_Internal(XrdSfsFileSystem *, XrdSysLogger *, const char *);
+
+public:
+
+   virtual XrdSfsDirectory *
+   newDir(char *user=0, int monid=0);
+
+   virtual XrdSfsFile *
+   newFile(char *user=0, int monid=0);
+
+   virtual int
+   chmod(const char             *Name,
+               XrdSfsMode        Mode,
+               XrdOucErrInfo    &out_error,
+         const XrdSecEntity     *client,
+         const char             *opaque = 0);
+
+   virtual int
+   exists(const char                *fileName,
+                XrdSfsFileExistence &exists_flag,
+                XrdOucErrInfo       &out_error,
+          const XrdSecEntity        *client,
+          const char                *opaque = 0);
+
+   virtual int
+   fsctl(const int               cmd,
+         const char             *args,
+               XrdOucErrInfo    &out_error,
+         const XrdSecEntity     *client);
+
+   virtual int
+   getStats(char *buff, int blen);
+
+   virtual const char *
+   getVersion();
+
+   virtual int
+   mkdir(const char             *dirName,
+               XrdSfsMode        Mode,
+               XrdOucErrInfo    &out_error,
+         const XrdSecEntity     *client,
+         const char             *opaque = 0);
+
+   virtual int
+   prepare(      XrdSfsPrep       &pargs,
+                 XrdOucErrInfo    &out_error,
+           const XrdSecEntity     *client = 0);
+
+   virtual int
+   rem(const char             *path,
+             XrdOucErrInfo    &out_error,
+       const XrdSecEntity     *client,
+       const char             *info = 0);
+
+   virtual int
+   remdir(const char             *dirName,
+                XrdOucErrInfo    &out_error,
+          const XrdSecEntity     *client,
+          const char             *info = 0);
+
+   virtual int
+   rename(const char             *oldFileName,
+          const char             *newFileName,
+                XrdOucErrInfo    &out_error,
+          const XrdSecEntity     *client,
+          const char             *infoO = 0,
+          const char             *infoN = 0);
+
+   virtual int
+   stat(const char             *Name,
+              struct stat      *buf,
+              XrdOucErrInfo    &out_error,
+        const XrdSecEntity     *client,
+        const char             *opaque = 0);
+
+   virtual int
+   stat(const char             *Name,
+              mode_t           &mode,
+              XrdOucErrInfo    &out_error,
+        const XrdSecEntity     *client,
+        const char             *opaque = 0);
+
+   virtual int
+   truncate(const char             *Name,
+                  XrdSfsFileOffset fileOffset,
+                  XrdOucErrInfo    &out_error,
+            const XrdSecEntity     *client = 0,
+            const char             *opaque = 0);
+
+   virtual int
+   Configure(XrdSysError &, XrdSfsFileSystem *native_fs);
+
+private:
+   static void
+   Initialize(      FileSystem      *&fs,
+                    XrdSfsFileSystem *native_fs,
+                    XrdSysLogger     *lp,
+              const char             *config_file);
+
+   FileSystem();
+
+   virtual
+  ~FileSystem();
+
+   int
+   xthrottle(XrdOucStream &Config);
+
+   int
+   xloadshed(XrdOucStream &Config);
+
+   int
+   xtrace(XrdOucStream &Config);
+
+   static FileSystem  *m_instance;
+   XrdSysError         m_eroute;
+   XrdOucTrace         m_trace;
+   std::string         m_config_file;
+   XrdSfsFileSystem   *m_sfs_ptr;
+   bool                m_initialized;
+   XrdThrottleManager  m_throttle;
+   XrdVersionInfo     *myVersion;
+
+};
+
+}
+
+#endif
+