From 847c8be8e499517db1cddd8e1d36094725cf18db Mon Sep 17 00:00:00 2001 From: Andrew Hanushevsky Date: Wed, 28 Jun 2017 15:47:29 -0700 Subject: [PATCH] *[Proxy]* Make cache I/O synchronization tunable. --- src/XrdOuc/XrdOucPsx.cc | 45 +++++++++ src/XrdOuc/XrdOucPsx.hh | 6 +- src/XrdPosix/XrdPosixConfig.cc | 9 ++ src/XrdPosix/XrdPosixFile.cc | 17 ++-- src/XrdPosix/XrdPosixXrootd.cc | 2 + src/XrdPss/XrdPss.hh | 6 -- src/XrdPss/XrdPssConfig.cc | 167 +++------------------------------ 7 files changed, 85 insertions(+), 167 deletions(-) diff --git a/src/XrdOuc/XrdOucPsx.cc b/src/XrdOuc/XrdOucPsx.cc index 06938e10302..cb4ecaff74f 100644 --- a/src/XrdOuc/XrdOucPsx.cc +++ b/src/XrdOuc/XrdOucPsx.cc @@ -285,6 +285,7 @@ bool XrdOucPsx::Parse(char *var, XrdOucStream &Config, XrdSysError &eDest) TS_Xeq("memcache", ParseCache); // Backward compatibility TS_Xeq("cache", ParseCache); TS_Xeq("cachelib", ParseCLib); + TS_Xeq("ciosync", ParseCio); TS_Xeq("inetmode", ParseINet); TS_Xeq("namelib", ParseNLib); TS_Xeq("setopt", ParseSet); @@ -458,6 +459,50 @@ char *XrdOucPsx::ParseCache(XrdSysError *Eroute, XrdOucStream &Config, char *pBu return val; } +/******************************************************************************/ +/* P a r s e C i o */ +/******************************************************************************/ + +/* Function: ParseCio + + Purpose: To parse the directive: ciosync + + the number of seconds between each sync attempt. + the maximum number of tries before giving up. + + Output: true upon success or false upon failure. +*/ + +bool XrdOucPsx::ParseCio(XrdSysError *Eroute, XrdOucStream &Config) +{ + char *val; + int tsec, mtry; + +// Get the try seconds +// + if (!(val = Config.GetWord()) || !val[0]) + {Eroute->Emsg("Config", "ciosync parameter not specified"); return false;} + +// Convert to seconds +// + if (XrdOuca2x::a2i(*Eroute,"ciosync interval",val,&tsec,10)) return false; + +// Get the max seconds +// + if (!(val = Config.GetWord()) || !val[0]) + {Eroute->Emsg("Config", "max time not specified"); return false;} + +// Convert to seconds +// + if (XrdOuca2x::a2i(*Eroute,"ciosync max time",val,&mtry,2)) return false; + +// Set values and return success +// + cioWait = tsec; + cioTries = mtry; + return true; +} + /******************************************************************************/ /* P a r s e C L i b */ /******************************************************************************/ diff --git a/src/XrdOuc/XrdOucPsx.hh b/src/XrdOuc/XrdOucPsx.hh index 4c363ffcab5..9f094282a62 100644 --- a/src/XrdOuc/XrdOucPsx.hh +++ b/src/XrdOuc/XrdOucPsx.hh @@ -52,6 +52,8 @@ bool ConfigSetup(XrdSysError &eDest, bool hush=false); bool ParseCache(XrdSysError *Eroute, XrdOucStream &Config); +bool ParseCio(XrdSysError *Eroute, XrdOucStream &Config); + bool ParseCLib(XrdSysError *Eroute, XrdOucStream &Config); bool ParseINet(XrdSysError *Eroute, XrdOucStream &Config); @@ -73,6 +75,8 @@ XrdOucTList *setLast; int maxRHCB; int traceLvl; int debugLvl; +int cioWait; +int cioTries; bool useV4; bool xLfn2Pfn; bool xPfn2Lfn; @@ -80,7 +84,7 @@ bool xPfn2Lfn; XrdOucPsx(XrdVersionInfo *vInfo, const char *cfn) : theN2N(0), theCache(0), theCache2(0), mCache(0), setFirst(0), setLast(0), maxRHCB(0), - traceLvl(0), debugLvl(0), + traceLvl(0), debugLvl(0), cioWait(0), cioTries(0), useV4(false), xLfn2Pfn(false), xPfn2Lfn(false), LocalRoot(0), RemotRoot(0), N2NLib(0), N2NParms(0), cPath(0), cParm(0), configFN(strdup(cfn)), diff --git a/src/XrdPosix/XrdPosixConfig.cc b/src/XrdPosix/XrdPosixConfig.cc index 6385d54b5a2..f708f70f527 100644 --- a/src/XrdPosix/XrdPosixConfig.cc +++ b/src/XrdPosix/XrdPosixConfig.cc @@ -64,6 +64,8 @@ extern XrdOucName2Name *theN2N; extern XrdCl::DirListFlags::Flags dlFlag; extern XrdSysLogger *theLogger; extern XrdSysTrace Trace; +extern int ddInterval; +extern int ddMaxTries; extern bool oidsOK; }; @@ -243,6 +245,13 @@ void XrdPosixConfig::SetConfig(XrdOucPsx &parms) // if (parms.maxRHCB > 0) XrdPosixFileRH::SetMax(parms.maxRHCB); +// Set delayed destro parameters if present +// + if (parms.cioWait > 0 && parms.cioTries > 0) + {XrdPosixGlobals::ddMaxTries = (parms.cioTries < 2 ? 2 : parms.cioTries); + XrdPosixGlobals::ddInterval = (parms.cioWait < 10 ? 10 : parms.cioWait); + } + // Handle the caching options // if (parms.theCache2) diff --git a/src/XrdPosix/XrdPosixFile.cc b/src/XrdPosix/XrdPosixFile.cc index b65d27f81ca..3d04bbfd4ad 100644 --- a/src/XrdPosix/XrdPosixFile.cc +++ b/src/XrdPosix/XrdPosixFile.cc @@ -54,6 +54,9 @@ namespace XrdPosixGlobals { extern XrdOucCache2 *theCache; extern XrdOucName2Name *theN2N; +extern int ddInterval; +extern int ddMaxTries; + int ddNumLost = 0; }; namespace @@ -146,9 +149,6 @@ void* XrdPosixFile::DelayedDestroy(void* vpf) // file cannot be closed in a clean fashion for some reason. // EPNAME("DDestroy"); - static const int ddInterval = 30; - static const int maxTries = (3*60)/ddInterval; - static int numLost = 0; XrdCl::XRootDStatus Status; const char *eTxt; @@ -159,7 +159,7 @@ void* XrdPosixFile::DelayedDestroy(void* vpf) // Wait for active I/O to complete // do{if (doWait) - {XrdSysTimer::Snooze(ddInterval); + {XrdSysTimer::Snooze(XrdPosixGlobals::ddInterval); doWait = false; } else { ddSem.Wait(); @@ -175,7 +175,8 @@ do{if (doWait) // Do some debugging // - DEBUG("DLY destory of "<numTries > maxTries) - {numLost++; ddCount--; + if (fCurr->numTries > XrdPosixGlobals::ddMaxTries) + {XrdPosixGlobals::ddNumLost++; ddCount--; DMSG("DDestroy", eTxt <<" timeout closing " <Origin() - <nextFile = ddLost; ddLost = fCurr; fCurr->Close(Status); diff --git a/src/XrdPosix/XrdPosixXrootd.cc b/src/XrdPosix/XrdPosixXrootd.cc index 715f956b5e9..954ccfa3e50 100644 --- a/src/XrdPosix/XrdPosixXrootd.cc +++ b/src/XrdPosix/XrdPosixXrootd.cc @@ -86,6 +86,8 @@ XrdCl::DirListFlags::Flags dlFlag = XrdCl::DirListFlags::None; XrdSysLogger *theLogger = 0; XrdSysTrace Trace("Posix", 0, (getenv("XRDPOSIX_DEBUG") ? TRACE_Debug : 0)); +int ddInterval= 30; +int ddMaxTries= 180/30; bool oidsOK = false; }; diff --git a/src/XrdPss/XrdPss.hh b/src/XrdPss/XrdPss.hh index 5f708bd620c..8e9249b2c2c 100644 --- a/src/XrdPss/XrdPss.hh +++ b/src/XrdPss/XrdPss.hh @@ -202,16 +202,10 @@ int ConfigProc(const char *ConfigFN); int ConfigXeq(char*, XrdOucStream&); const char *getDomain(const char *hName); -int xcach(XrdSysError *Eroute, XrdOucStream &Config); -int xcacl(XrdSysError *Eroute, XrdOucStream &Config); int xconf(XrdSysError *Eroute, XrdOucStream &Config); int xdef( XrdSysError *Eroute, XrdOucStream &Config); int xexp( XrdSysError *Eroute, XrdOucStream &Config); -int xinet(XrdSysError *errp, XrdOucStream &Config); int xperm(XrdSysError *errp, XrdOucStream &Config); int xorig(XrdSysError *errp, XrdOucStream &Config); -int xsopt(XrdSysError *Eroute, XrdOucStream &Config); -int xtrac(XrdSysError *Eroute, XrdOucStream &Config); -int xnml (XrdSysError *Eroute, XrdOucStream &Config); }; #endif diff --git a/src/XrdPss/XrdPssConfig.cc b/src/XrdPss/XrdPssConfig.cc index df73b2012d1..fb298086950 100644 --- a/src/XrdPss/XrdPssConfig.cc +++ b/src/XrdPss/XrdPssConfig.cc @@ -80,6 +80,9 @@ #define TS_Xeq(x,m) if (!strcmp(x,var)) return m(&eDest, Config); +#define TS_PSX(x,m) if (!strcmp(x,var)) \ + return (psxConfig->m(&eDest, Config) ? 0 : 1); + /*******x**********************************************************************/ /* G l o b a l s */ /******************************************************************************/ @@ -436,18 +439,19 @@ int XrdPssSys::ConfigXeq(char *var, XrdOucStream &Config) // Process items. for either a local or a remote configuration // - TS_Xeq("memcache", xcach); // Backward compatibility - TS_Xeq("cache", xcach); - TS_Xeq("cachelib", xcacl); + TS_PSX("namelib", ParseNLib); + TS_PSX("memcache", ParseCache); // Backward compatibility + TS_PSX("cache", ParseCache); + TS_PSX("cachelib", ParseCLib); + TS_PSX("ciosync", ParseCio); TS_Xeq("config", xconf); - TS_Xeq("inetmode", xinet); - TS_Xeq("origin", xorig); - TS_Xeq("permit", xperm); - TS_Xeq("setopt", xsopt); - TS_Xeq("trace", xtrac); - TS_Xeq("namelib", xnml); TS_Xeq("defaults", xdef); TS_Xeq("export", xexp); + TS_PSX("inetmode", ParseINet); + TS_Xeq("origin", xorig); + TS_Xeq("permit", xperm); + TS_PSX("setopt", ParseSet); + TS_PSX("trace", ParseTrace); // Copy the variable name as this may change because it points to an // internal buffer in Config. The vagaries of effeciency. Then get value. @@ -481,59 +485,6 @@ const char *XrdPssSys::getDomain(const char *hName) return hName; } -/******************************************************************************/ -/* x c a c h */ -/******************************************************************************/ - -/* Function: xcach - - Purpose: To parse the directive: cache [...] - - is one of the following: - debug {0 | 1 | 2} - logstats enables stats logging - max2cache largest read to cache (can be suffixed with k, m, g). - minpages smallest number of pages allowed (default 256) - mode {r | w} - pagesize size of each cache page (can be suffixed with k, m, g). - preread [minpages [minrdsz]] [perf nn [recalc]] - r/w enables caching for files opened read/write. - sfiles {on | off | .} - size size of cache in bytes (can be suffixed with k, m, g). - - Output: 0 upon success or 1 upon failure. -*/ - -int XrdPssSys::xcach(XrdSysError *Eroute, XrdOucStream &Config) -{ - -// We parse this using the configurator -// - return (psxConfig->ParseCache(Eroute, Config) ? 0 : 1); -} - -/******************************************************************************/ -/* x c a c l */ -/******************************************************************************/ - -/* Function: xcacl - - Purpose: To parse the directive: cachelib {|default} [] - - the path of the cache library to be used. - optional parms to be passed - - Output: 0 upon success or !0 upon failure. -*/ - -int XrdPssSys::xcacl(XrdSysError *Eroute, XrdOucStream &Config) -{ - -// We parse this using the configurator -// - return (psxConfig->ParseCLib(Eroute, Config) ? 0 : 1); -} - /******************************************************************************/ /* x c o n f */ /******************************************************************************/ @@ -543,7 +494,8 @@ int XrdPssSys::xcacl(XrdSysError *Eroute, XrdOucStream &Config) Purpose: To parse the directive: config is one of the following: - workers number of queue workers > 0 + streams number of i/o streams + workers number of queue workers Output: 0 upon success or 1 upon failure. */ @@ -631,51 +583,6 @@ int XrdPssSys::xexp(XrdSysError *Eroute, XrdOucStream &Config) if (*(pP->Path()) == '*') XrdPosixConfig::setOids(true); return 0; } - -/******************************************************************************/ -/* x i n e t */ -/******************************************************************************/ - -/* Function: xinet - - Purpose: To parse the directive: inetmode v4 | v6 - - v4 use only IPV4 addresses to connect to servers. - v6 use IPV4 mapped addresses or IPV6 addresses, as needed. - - Output: 0 upon success or !0 upon failure. -*/ - -int XrdPssSys::xinet(XrdSysError *Eroute, XrdOucStream &Config) -{ - -// We parse this using the configurator -// - return (psxConfig->ParseINet(Eroute, Config) ? 0 : 1); -} - -/******************************************************************************/ -/* x n m l */ -/******************************************************************************/ - -/* Function: xnml - - Purpose: To parse the directive: namelib [] pfn [] - - one or more: [-lfn2pfn] [-lfncache] - the path of the filesystem library to be used. - optional parms to be passed - - Output: 0 upon success or !0 upon failure. -*/ - -int XrdPssSys::xnml(XrdSysError *Eroute, XrdOucStream &Config) -{ - -// We parse this using the configurator -// - return (psxConfig->ParseNLib(Eroute, Config) ? 0 : 1); -} /******************************************************************************/ /* x o r i g */ @@ -798,47 +705,3 @@ do {if (!(val = Config.GetWord())) return 0; } - -/******************************************************************************/ -/* x s o p t */ -/******************************************************************************/ - -/* Function: xsopt - - Purpose: To parse the directive: setopt - - is an XrdClient option keyword. - is the value the option is to have. - - Output: 0 upon success or !0 upon failure. -*/ - -int XrdPssSys::xsopt(XrdSysError *Eroute, XrdOucStream &Config) -{ - -// We parse this using the configurator -// - return (psxConfig->ParseSet(Eroute, Config) ? 0 : 1); -} - -/******************************************************************************/ -/* x t r a c e */ -/******************************************************************************/ - -/* Function: xtrace - - Purpose: To parse the directive: trace - - the blank separated list of events to trace. Trace - directives are cummalative. - - Output: retc upon success or -EINVAL upon failure. -*/ - -int XrdPssSys::xtrac(XrdSysError *Eroute, XrdOucStream &Config) -{ - -// We parse this using the configurator -// - return (psxConfig->ParseTrace(Eroute, Config) ? 0 : 1); -}