From 1339e37849054c363cdc8f22e8beb0d27d67f521 Mon Sep 17 00:00:00 2001 From: Andrew Hanushevsky Date: Wed, 23 Oct 2019 20:48:31 -0700 Subject: [PATCH] [Proxy] Refactor proxy server caching implementation. --- src/XrdFileCache/XrdFileCache.cc | 27 +- src/XrdFileCache/XrdFileCache.hh | 26 +- src/XrdFileCache/XrdFileCacheConfiguration.cc | 14 +- src/XrdFileCache/XrdFileCacheFile.hh | 2 +- src/XrdFileCache/XrdFileCacheIO.cc | 8 +- src/XrdFileCache/XrdFileCacheIO.hh | 20 +- src/XrdFileCache/XrdFileCacheIOEntireFile.cc | 8 +- src/XrdFileCache/XrdFileCacheIOEntireFile.hh | 8 +- src/XrdFileCache/XrdFileCacheIOFileBlock.cc | 10 +- src/XrdFileCache/XrdFileCacheIOFileBlock.hh | 8 +- src/XrdHeaders.cmake | 1 - src/XrdOuc/XrdOucCache.hh | 737 +++++++++++------- src/XrdOuc/XrdOucCache2.hh | 390 --------- src/XrdOuc/XrdOucPsx.cc | 31 +- src/XrdOuc/XrdOucPsx.hh | 6 +- src/XrdPosix.cmake | 1 - src/XrdPosix/XrdPosixCache.cc | 10 +- src/XrdPosix/XrdPosixCacheBC.hh | 136 ---- src/XrdPosix/XrdPosixConfig.cc | 49 +- src/XrdPosix/XrdPosixFile.cc | 59 +- src/XrdPosix/XrdPosixFile.hh | 13 +- src/XrdPosix/XrdPosixPrepIO.hh | 8 +- src/XrdPosix/XrdPosixXrootd.cc | 36 +- src/XrdPosix/XrdPosixXrootd.hh | 6 - src/XrdPss/XrdPssConfig.cc | 17 +- src/XrdRmc/XrdRmc.cc | 52 ++ src/XrdRmc/XrdRmc.hh | 151 ++++ src/XrdRmc/XrdRmcData.cc | 581 ++++++++++++++ src/XrdRmc/XrdRmcData.hh | 148 ++++ src/XrdRmc/XrdRmcReal.cc | 599 ++++++++++++++ src/XrdRmc/XrdRmcReal.hh | 134 ++++ src/XrdRmc/XrdRmcSlot.hh | 154 ++++ src/XrdSfs/XrdSfsInterface.hh | 35 +- src/XrdUtils.cmake | 14 +- src/XrdVersionPlugin.hh | 5 + 35 files changed, 2458 insertions(+), 1046 deletions(-) delete mode 100644 src/XrdOuc/XrdOucCache2.hh delete mode 100644 src/XrdPosix/XrdPosixCacheBC.hh create mode 100644 src/XrdRmc/XrdRmc.cc create mode 100644 src/XrdRmc/XrdRmc.hh create mode 100644 src/XrdRmc/XrdRmcData.cc create mode 100644 src/XrdRmc/XrdRmcData.hh create mode 100644 src/XrdRmc/XrdRmcReal.cc create mode 100644 src/XrdRmc/XrdRmcReal.hh create mode 100644 src/XrdRmc/XrdRmcSlot.hh diff --git a/src/XrdFileCache/XrdFileCache.cc b/src/XrdFileCache/XrdFileCache.cc index 5d2a591015d..305d2dead94 100644 --- a/src/XrdFileCache/XrdFileCache.cc +++ b/src/XrdFileCache/XrdFileCache.cc @@ -67,13 +67,17 @@ void *PrefetchThread(void* ptr) extern "C" { -XrdOucCache2 *XrdOucGetCache2(XrdSysLogger *logger, - const char *config_filename, - const char *parameters) +XrdOucCache *XrdOucGetCache(XrdSysLogger *logger, + const char *config_filename, + const char *parameters, + XrdOucEnv *envP) { XrdSysError err(logger, ""); err.Say("++++++ Proxy file cache initialization started."); + if (envP) + XrdFileCache::Cache::schedP = (XrdScheduler *)envP->GetPtr("XrdScheduler*"); + Cache &factory = Cache::CreateInstance(logger); if (! factory.Config(config_filename, parameters)) @@ -160,7 +164,7 @@ bool Cache::Decide(XrdOucCacheIO* io) } Cache::Cache(XrdSysLogger *logger) : - XrdOucCache2(), + XrdOucCache(), m_log(logger, "XrdFileCache_"), m_trace(new XrdSysTrace("XrdFileCache", logger)), m_traceID("Manager"), @@ -178,7 +182,7 @@ Cache::Cache(XrdSysLogger *logger) : } -XrdOucCacheIO2 *Cache::Attach(XrdOucCacheIO2 *io, int Options) +XrdOucCacheIO *Cache::Attach(XrdOucCacheIO *io, int Options) { const char* tpfx = "Cache::Attach() "; @@ -221,13 +225,6 @@ XrdOucCacheIO2 *Cache::Attach(XrdOucCacheIO2 *io, int Options) } -int Cache::isAttached() -{ - // virutal function of XrdOucCache, don't see it used in pfc or posix layer - return true; -} - - void Cache::AddWriteTask(Block* b, bool fromRead) { TRACE(Dump, "Cache::AddWriteTask() bOff=%ld " << b->m_offset); @@ -675,7 +672,7 @@ void Cache::Prefetch() //============================================================================== -//=== Virtuals from XrdOucCache2 +//=== Virtuals from XrdOucCache //============================================================================== //------------------------------------------------------------------------------ @@ -801,7 +798,7 @@ int Cache::LocalFilePath(const char *curl, char *buff, int blen, //! @return <0 Error has occurred, return value is -errno; fail open request. //! =0 Continue with open() request. //! >0 Defer open but treat the file as actually being open. Use the -//! XrdOucCacheIO2::Open() method to open the file at a later time. +//! XrdOucCacheIO::Open() method to open the file at a later time. //------------------------------------------------------------------------------ int Cache::Prepare(const char *curl, int oflags, mode_t mode) @@ -853,7 +850,7 @@ int Cache::Prepare(const char *curl, int oflags, mode_t mode) } //______________________________________________________________________________ -// virtual method of XrdOucCache2. +// virtual method of XrdOucCache. //! //! @return <0 - Stat failed, value is -errno. //! =0 - Stat succeeded, sbuff holds stat information. diff --git a/src/XrdFileCache/XrdFileCache.hh b/src/XrdFileCache/XrdFileCache.hh index b5c6fc52a69..cd509a50f17 100644 --- a/src/XrdFileCache/XrdFileCache.hh +++ b/src/XrdFileCache/XrdFileCache.hh @@ -25,7 +25,7 @@ #include "Xrd/XrdScheduler.hh" #include "XrdVersion.hh" #include "XrdSys/XrdSysPthread.hh" -#include "XrdOuc/XrdOucCache2.hh" +#include "XrdOuc/XrdOucCache.hh" #include "XrdOuc/XrdOucCallBack.hh" #include "XrdCl/XrdClDefaultEnv.hh" @@ -276,7 +276,7 @@ struct PathTokenizer : private SplitParser //---------------------------------------------------------------------------- //! Attaches/creates and detaches/deletes cache-io objects for disk based cache. //---------------------------------------------------------------------------- -class Cache : public XrdOucCache2 +class Cache : public XrdOucCache { public: //--------------------------------------------------------------------- @@ -287,30 +287,21 @@ public: //--------------------------------------------------------------------- //! Obtain a new IO object that fronts existing XrdOucCacheIO. //--------------------------------------------------------------------- - using XrdOucCache2::Attach; + using XrdOucCache::Attach; - virtual XrdOucCacheIO2 *Attach(XrdOucCacheIO2 *, int Options = 0); + virtual XrdOucCacheIO *Attach(XrdOucCacheIO *, int Options = 0); //--------------------------------------------------------------------- - //! Number of cache-io objects atteched through this cache. - //--------------------------------------------------------------------- - virtual int isAttached(); - - //--------------------------------------------------------------------- - // Virtual function of XrdOucCache2. Used to pass environmental info. - virtual void EnvInfo(XrdOucEnv &theEnv); - - //--------------------------------------------------------------------- - // Virtual function of XrdOucCache2. Used for redirection to a local + // Virtual function of XrdOucCache. Used for redirection to a local // file on a distributed FS. virtual int LocalFilePath(const char *url, char *buff=0, int blen=0, LFP_Reason why=ForAccess, bool forall=false); //--------------------------------------------------------------------- - // Virtual function of XrdOucCache2. Used for deferred open. + // Virtual function of XrdOucCache. Used for deferred open. virtual int Prepare(const char *url, int oflags, mode_t mode); - // virtual function of XrdOucCache2. + // virtual function of XrdOucCache. virtual int Stat(const char *url, struct stat &sbuff); // virtual function of XrdOucCache. @@ -409,6 +400,8 @@ public: void ExecuteCommandUrl(const std::string& command_url); + static XrdScheduler *schedP; + private: bool ConfigParameters(std::string, XrdOucStream&, TmpConfiguration &tmpc); bool ConfigXeq(char *, XrdOucStream &); @@ -420,7 +413,6 @@ private: int UnlinkCommon(const std::string& f_name, bool fail_if_open); static Cache *m_factory; //!< this object - static XrdScheduler *schedP; XrdSysError m_log; //!< XrdFileCache namespace logger XrdSysTrace *m_trace; diff --git a/src/XrdFileCache/XrdFileCacheConfiguration.cc b/src/XrdFileCache/XrdFileCacheConfiguration.cc index 8dfda907cad..30a5b0a5675 100644 --- a/src/XrdFileCache/XrdFileCacheConfiguration.cc +++ b/src/XrdFileCache/XrdFileCacheConfiguration.cc @@ -18,7 +18,7 @@ using namespace XrdFileCache; -XrdVERSIONINFO(XrdOucGetCache2, XrdFileCache); +XrdVERSIONINFO(XrdOucGetCache, XrdFileCache); bool Cache::cfg2bytes(const std::string &str, long long &store, long long totalSpace, const char *name) { @@ -174,7 +174,7 @@ bool Cache::Config(const char *config_filename, const char *parameters) // Obtain OFS configurator for OSS plugin. XrdOfsConfigPI *ofsCfg = XrdOfsConfigPI::New(config_filename,&Config,&m_log, - &XrdVERSIONINFOVAR(XrdOucGetCache2)); + &XrdVERSIONINFOVAR(XrdOucGetCache)); if (! ofsCfg) return false; TmpConfiguration tmpc; @@ -650,13 +650,3 @@ bool Cache::ConfigParameters(std::string part, XrdOucStream& config, TmpConfigur return true; } - -//______________________________________________________________________________ - - -void Cache::EnvInfo(XrdOucEnv &theEnv) -{ -// Extract out the pointer to the scheduler -// - schedP = (XrdScheduler *) theEnv.GetPtr("XrdScheduler*"); -} diff --git a/src/XrdFileCache/XrdFileCacheFile.hh b/src/XrdFileCache/XrdFileCacheFile.hh index 51420fdb0f6..8e9e8ae632e 100644 --- a/src/XrdFileCache/XrdFileCacheFile.hh +++ b/src/XrdFileCache/XrdFileCacheFile.hh @@ -21,7 +21,7 @@ #include "XrdCl/XrdClXRootDResponses.hh" #include "XrdCl/XrdClDefaultEnv.hh" -#include "XrdOuc/XrdOucCache2.hh" +#include "XrdOuc/XrdOucCache.hh" #include "XrdOuc/XrdOucIOVec.hh" #include "XrdFileCacheInfo.hh" diff --git a/src/XrdFileCache/XrdFileCacheIO.cc b/src/XrdFileCache/XrdFileCacheIO.cc index b0ef2e43a10..f4eba249961 100644 --- a/src/XrdFileCache/XrdFileCacheIO.cc +++ b/src/XrdFileCache/XrdFileCacheIO.cc @@ -3,13 +3,13 @@ using namespace XrdFileCache; -IO::IO(XrdOucCacheIO2 *io, XrdOucCacheStats &stats, Cache &cache) : +IO::IO(XrdOucCacheIO *io, XrdOucCacheStats &stats, Cache &cache) : m_statsGlobal(stats), m_cache(cache), m_traceID("IO"), m_io(io) { m_path = m_io->Path(); } -void IO::Update(XrdOucCacheIO2 &iocp) +void IO::Update(XrdOucCacheIO &iocp) { SetInput(&iocp); TRACE_PC(Info, const char* loc = m_io->Location(), @@ -17,13 +17,13 @@ void IO::Update(XrdOucCacheIO2 &iocp) ((loc && loc[0] != 0) ? loc : "")); } -void IO::SetInput(XrdOucCacheIO2* x) +void IO::SetInput(XrdOucCacheIO* x) { XrdSysMutexHelper lock(&updMutex); m_io = x; } -XrdOucCacheIO2* IO::GetInput() +XrdOucCacheIO* IO::GetInput() { XrdSysMutexHelper lock(&updMutex); return m_io; diff --git a/src/XrdFileCache/XrdFileCacheIO.hh b/src/XrdFileCache/XrdFileCacheIO.hh index 77a7fc5f94e..538e88e37ea 100644 --- a/src/XrdFileCache/XrdFileCacheIO.hh +++ b/src/XrdFileCache/XrdFileCacheIO.hh @@ -4,7 +4,7 @@ class XrdSysTrace; #include "XrdFileCache.hh" -#include "XrdOuc/XrdOucCache2.hh" +#include "XrdOuc/XrdOucCache.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdSys/XrdSysPthread.hh" @@ -13,10 +13,10 @@ namespace XrdFileCache //---------------------------------------------------------------------------- //! Base cache-io class that implements XrdOucCacheIO abstract methods. //---------------------------------------------------------------------------- -class IO : public XrdOucCacheIO2 +class IO : public XrdOucCacheIO { public: - IO (XrdOucCacheIO2 *io, XrdOucCacheStats &stats, Cache &cache); + IO (XrdOucCacheIO *io, XrdOucCacheStats &stats, Cache &cache); //! Original data source. virtual XrdOucCacheIO *Base() { return m_io; } @@ -24,23 +24,23 @@ public: //! Original data source URL. virtual const char *Path() { return m_io->Path(); } - using XrdOucCacheIO2::Sync; + using XrdOucCacheIO::Sync; virtual int Sync() { return 0; } - using XrdOucCacheIO2::Trunc; + using XrdOucCacheIO::Trunc; virtual int Trunc(long long Offset) { return -ENOTSUP; } - using XrdOucCacheIO2::Write; + using XrdOucCacheIO::Write; virtual int Write(char *Buffer, long long Offset, int Length) { return -ENOTSUP; } - virtual void Update(XrdOucCacheIO2 &iocp); + virtual void Update(XrdOucCacheIO &iocp); XrdSysTrace* GetTrace() { return m_cache.GetTrace(); } - XrdOucCacheIO2* GetInput(); + XrdOucCacheIO* GetInput(); protected: XrdOucCacheStats &m_statsGlobal; //!< reference to Cache statistics @@ -51,9 +51,9 @@ protected: const char* GetPath() { return m_path.c_str(); } private: - XrdOucCacheIO2 *m_io; //!< original data source + XrdOucCacheIO *m_io; //!< original data source XrdSysMutex updMutex; - void SetInput(XrdOucCacheIO2*); + void SetInput(XrdOucCacheIO*); }; } diff --git a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc index 2e7d5d9dd23..3d2ca7039ec 100644 --- a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc +++ b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc @@ -32,7 +32,7 @@ using namespace XrdFileCache; //______________________________________________________________________________ -IOEntireFile::IOEntireFile(XrdOucCacheIO2 *io, XrdOucCacheStats &stats, Cache & cache) : +IOEntireFile::IOEntireFile(XrdOucCacheIO *io, XrdOucCacheStats &stats, Cache & cache) : IO(io, stats, cache), m_file(0), m_localStat(0) @@ -135,7 +135,7 @@ bool IOEntireFile::ioActive() } //______________________________________________________________________________ -XrdOucCacheIO *IOEntireFile::Detach() +bool IOEntireFile::Detach(XrdOucCacheIOCD &iocdP) { // Called from XrdPosixFile destructor. @@ -147,9 +147,9 @@ XrdOucCacheIO *IOEntireFile::Detach() m_file->RequestSyncOfDetachStats(); Cache::GetInstance().ReleaseFile(m_file, this); } - XrdOucCacheIO *io = GetInput(); delete this; - return io; +//???? This needs to be coordinated ith old ioActive() and return false if active + return true; } //______________________________________________________________________________ diff --git a/src/XrdFileCache/XrdFileCacheIOEntireFile.hh b/src/XrdFileCache/XrdFileCacheIOEntireFile.hh index 1e4894fcdf4..f21ae983abd 100644 --- a/src/XrdFileCache/XrdFileCacheIOEntireFile.hh +++ b/src/XrdFileCache/XrdFileCacheIOEntireFile.hh @@ -43,7 +43,7 @@ public: //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ - IOEntireFile(XrdOucCacheIO2 *io, XrdOucCacheStats &stats, Cache &cache); + IOEntireFile(XrdOucCacheIO *io, XrdOucCacheStats &stats, Cache &cache); //------------------------------------------------------------------------ //! Destructor @@ -64,7 +64,7 @@ public: //! //! @return number of bytes read //--------------------------------------------------------------------- - using XrdOucCacheIO2::Read; + using XrdOucCacheIO::Read; virtual int Read(char *Buffer, long long Offset, int Length); @@ -76,7 +76,7 @@ public: //! //! @return total bytes read //--------------------------------------------------------------------- - using XrdOucCacheIO2::ReadV; + using XrdOucCacheIO::ReadV; virtual int ReadV(const XrdOucIOVec *readV, int n); @@ -85,7 +85,7 @@ public: //! //! @return original source \ref XrdPosixFile //--------------------------------------------------------------------- - virtual XrdOucCacheIO* Detach(); + virtual bool Detach(XrdOucCacheIOCD &iocdP); //! \brief Virtual method of XrdOucCacheIO. //! Called to check if destruction needs to be done in a separate task. diff --git a/src/XrdFileCache/XrdFileCacheIOFileBlock.cc b/src/XrdFileCache/XrdFileCacheIOFileBlock.cc index 3d4c2b1e94d..dc50b65149e 100644 --- a/src/XrdFileCache/XrdFileCacheIOFileBlock.cc +++ b/src/XrdFileCache/XrdFileCacheIOFileBlock.cc @@ -36,7 +36,7 @@ using namespace XrdFileCache; //______________________________________________________________________________ -IOFileBlock::IOFileBlock(XrdOucCacheIO2 *io, XrdOucCacheStats &statsGlobal, Cache & cache) : +IOFileBlock::IOFileBlock(XrdOucCacheIO *io, XrdOucCacheStats &statsGlobal, Cache & cache) : IO(io, statsGlobal, cache), m_localStat(0), m_info(cache.GetTrace(), false), m_infoFile(0) { m_blocksize = Cache::GetInstance().RefConfiguration().m_hdfsbsize; @@ -54,9 +54,9 @@ IOFileBlock::~IOFileBlock() } //______________________________________________________________________________ -XrdOucCacheIO* IOFileBlock::Detach() +bool IOFileBlock::Detach(XrdOucCacheIOCD &iocdP) { - // Called from XrdPosixFile destructor + // Called from XrdPosixFile destructor ???? Not really tue TRACEIO(Info, "Detach IOFileBlock"); @@ -72,9 +72,9 @@ XrdOucCacheIO* IOFileBlock::Detach() } } } - XrdOucCacheIO *io = GetInput(); delete this; - return io; +//???? This needs to be coordinated with old ioActive() and return false if active + return true; } diff --git a/src/XrdFileCache/XrdFileCacheIOFileBlock.hh b/src/XrdFileCache/XrdFileCacheIOFileBlock.hh index 9b42997aeab..3b96642ffad 100644 --- a/src/XrdFileCache/XrdFileCacheIOFileBlock.hh +++ b/src/XrdFileCache/XrdFileCacheIOFileBlock.hh @@ -20,7 +20,7 @@ #include #include -#include "XrdOuc/XrdOucCache2.hh" +#include "XrdOuc/XrdOucCache.hh" #include "XrdSys/XrdSysPthread.hh" #include "XrdFileCacheIO.hh" @@ -41,7 +41,7 @@ public: //------------------------------------------------------------------------ //! Constructor. //------------------------------------------------------------------------ - IOFileBlock(XrdOucCacheIO2 *io, XrdOucCacheStats &stats, Cache &cache); + IOFileBlock(XrdOucCacheIO *io, XrdOucCacheStats &stats, Cache &cache); //------------------------------------------------------------------------ //! Destructor. @@ -53,12 +53,12 @@ public: //! //! @return original source \ref XrdPosixFile //--------------------------------------------------------------------- - virtual XrdOucCacheIO *Detach(); + virtual bool Detach(XrdOucCacheIOCD &iocdP); //--------------------------------------------------------------------- //! Pass Read request to the corresponding File object. //--------------------------------------------------------------------- - using XrdOucCacheIO2::Read; + using XrdOucCacheIO::Read; virtual int Read(char *Buffer, long long Offset, int Length); diff --git a/src/XrdHeaders.cmake b/src/XrdHeaders.cmake index 852cf68bee1..7017f66c1b9 100644 --- a/src/XrdHeaders.cmake +++ b/src/XrdHeaders.cmake @@ -24,7 +24,6 @@ set( XROOTD_PUBLIC_HEADERS XrdNet/XrdNetSocket.hh XrdOuc/XrdOucBuffer.hh XrdOuc/XrdOucCRC.hh - XrdOuc/XrdOucCache.hh XrdOuc/XrdOucCacheCM.hh XrdOuc/XrdOucCacheStats.hh XrdOuc/XrdOucCallBack.hh diff --git a/src/XrdOuc/XrdOucCache.hh b/src/XrdOuc/XrdOucCache.hh index 287b1833024..64997d1934f 100644 --- a/src/XrdOuc/XrdOucCache.hh +++ b/src/XrdOuc/XrdOucCache.hh @@ -4,7 +4,7 @@ /* */ /* X r d O u c C a c h e . h h */ /* */ -/* (c) 2011 by the Board of Trustees of the Leland Stanford, Jr., University */ +/* (c) 2019 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ @@ -30,56 +30,13 @@ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ +#include + #include "XrdOuc/XrdOucCacheStats.hh" #include "XrdOuc/XrdOucIOVec.hh" -/* The classes defined here can be used to implement a general cache for - data from an arbitrary source (e.g. files, sockets, etc); as follows: - - 1. Create an instance of XrdOucCacheIO. This object is used to actually - bring in missing data into the cache or write out dirty cache pages. - There can be many instances of this class, as needed. However, make sure - that there is a 1-to-1 unique correspondence between data and its CacheIO - object. Violating this may cause the same data to be cached multiple - times and if the cache is writable the data may be inconsistent! - - 2. Create an instance of XrdOucCache. You can specify various cache - handling parameters (see the class definition). You can also define - additional instances if you want more than one cache. The specific cache - you create will be defined by an implementation that derives from these - classes. For instance, an implementation of a memory cache is defined - in "XrdOucCacheDram.hh". - - 3. Use the Attach() method in XrdOucCache to attach your XrdOucCacheIO - object with a cache instance. The method returns a remanufactured - XrdOucCacheIO object that interposes the cache in front of the original - XrdOucCacheIO. This allows you to transparently use the cache. - - 4. When finished using the remanufactured XrdOucCacheIO object, use its - Detach() method to remove the association from the cache. Other actions - are defined by the actual implementation. For instance XrdOucCacheDram - also releases any assigned cache pages, writes out any dirty pages, and - may optionally delete the object when all references have been removed. - - 5. You may delete cache instances as well. Just be sure that no associations - still exist using the XrdOucCache::isAttached() method. Otherwise, the - cache destructor will wait until all attached objects are detached. - - Example: - class physIO : public XrdOucCacheIO {...}; // Define required methods - class xCache : public XrdOucCache {...}; // The cache implementation - XrdOucCache::Parms myParms; // Set any desired parameters - XrdOucCache *myCache; - XrdOucCacheIO *cacheIO; - xCache theCache; // Implementation instance - - myCache = theCache.Create(myParms); // Create a cache instance - cacheIO = myCache->Attach(physIO); // Interpose the cache - - // Use cacheIO (fronted by myCache) instead of physIO. When done... - - delete cacheIO->Detach(); // Deletes cacheIO and physIO -*/ +struct stat; +class XrdOucEnv; /******************************************************************************/ /* X r d O u c C a c h e I O C B */ @@ -109,163 +66,270 @@ virtual ~XrdOucCacheIOCB() {} }; /******************************************************************************/ -/* C l a s s X r d O u c C a c h e I O */ +/* X r d O u c C a c h e I O C D */ /******************************************************************************/ -/* The XrdOucCacheIO object is responsible for interacting with the original - data source/target. It can be used with or without a front-end cache. +//----------------------------------------------------------------------------- +//! The XrdOucCacheIOCD defines a callback object that is used to handle +//! XrdOucCacheIO::Detach() requests. It is passed to Detach() and if Detach() +//! returns false then the DetachDone() method must be called when the object +//! has been successfully detached from the cache. +//----------------------------------------------------------------------------- - Six abstract methods are provided FSize(), Path(), Read(), Sync(), Trunc(), - and Write(). You must provide implementations for each as described below. +class XrdOucCacheIOCD +{ +public: - Four additional virtual methods are pre-defined: Base(), Detach(), and - Preread() (2x). Normally, there is no need to over-ride these methods. +//------------------------------------------------------------------------------ +//! Indicate that the CacheIO object has been detached. +//------------------------------------------------------------------------------ +virtual +void DetachDone() = 0; - Finally, each object carries with it a XrdOucCacheStats object. -*/ + XrdOucCacheIOCD() {} +virtual ~XrdOucCacheIOCD() {} +}; + +/******************************************************************************/ +/* C l a s s X r d O u c C a c h e I O */ +/******************************************************************************/ + +//------------------------------------------------------------------------------ +//! The XrdOucCacheIO object is responsible for interacting with the original +//! data source/target. It can be used with or without a front-end cache. +//------------------------------------------------------------------------------ class XrdOucCacheIO { public: -// FSize() returns the current size of the file associated with this object. +//------------------------------------------------------------------------------ +//! Detach this CacheIO object from the cache. +//! +//! @note This method must be called instead of using the delete operator +//! since CacheIO objects may have multiple outstanding references and +//! actual deletion may need to be defered. +//! +//! @param iocd reference to the detach complete callback object. +//! +//! @return true Deletion can occur immediately. There is no outstanding I/O. +//! @return false Deletion must be deferred until it is safe to so from the +//! cache perspective. At which point, the cache will call the +//! DetachDone() method in the passed callback object. No locks +//! may be held with respect to the CacheIO object when this is +//! done to avoid deadlocks. +//------------------------------------------------------------------------------ + +virtual bool Detach(XrdOucCacheIOCD &iocd) = 0; -// Success: size of the file in bytes. -// Failure: -errno associated with the error. +//------------------------------------------------------------------------------ +//! Obtain size of the file. +//! +//! @return Size of the file in bytes. +//------------------------------------------------------------------------------ virtual -long long FSize() = 0; +long long FSize() = 0; -// Path() returns the path name associated with this object. -// -virtual -const char *Path() = 0; +//------------------------------------------------------------------------------ +//! Perform an fstat() operation (defaults to passthrough). +//! +//! @param sbuff reference to the stat buffer to be filled in. Only fields +//! st_size, st_blocks, st_mtime (st_atime and st_ctime may be +//! set to st_mtime), st_ino, and st_mode need to be set. All +//! other fields are preset and should not be changed. +//! +//! @return <0 - fstat failed, value is -errno. +//! =0 - fstat succeeded, sbuff holds stat information. +//! >0 - fstat could not be done, forward operation to next level. +//------------------------------------------------------------------------------ -// Read() places Length bytes in Buffer from a data source at Offset. -// When fronted by a cache, the cache is inspected first. +virtual int Fstat(struct stat &sbuff) {(void)sbuff; return 1;} -// Success: actual number of bytes placed in Buffer. -// Failure: -errno associated with the error. +//----------------------------------------------------------------------------- +//! Get the file's location (i.e. endpoint hostname and port) +//! +//! @return A pointer to the file's location. It remains valid until the file +//! is closed. A null string means the file is not open or is unknown. +//----------------------------------------------------------------------------- virtual -int Read (char *Buffer, long long Offset, int Length) = 0; +const char *Location() {return "";} -// ReadV() Performs a vector of read requests. When fronted by a cache, -// the cache is inspected first. By batching requests, it provides -// us the ability to skip expensive network round trips. -// If any reads fail or return short, the entire operation should -// fail. - -// Success: actual number of bytes read. -// Failure: -errno associated with the read error. -virtual -int ReadV(const XrdOucIOVec *readV, int n) - {int nbytes = 0, curCount = 0; - for (int i=0; iDetach()" if you want to make sure you -// delete the underlying object as well. Alternatively, use the optADB -// option when attaching a CacheIO object to a cache. This will delete -// underlying object and always return 0 to avoid a double delete. -// When not fronted by a cache, Detach() always returns itself. This -// makes its use consistent whether or not a cache is employed. -// -virtual XrdOucCacheIO *Detach() {return this;} - - -// ioActive() returns true if there is any ongoing IO operation. The function is -// used in XrdPosixXrootd::Close() to check if destruction od PosixFile -// has to be done in a separate task. -virtual bool ioActive() { return false; } - -// Preread() places Length bytes into the cache from a data source at Offset. -// When there is no cache or the associated cache does not support or -// allow pre-reads, it's a no-op. Cache placement limits do not apply. -// To maximize parallelism, Peread() should called *after* obtaining -// the wanted bytes using Read(). If the cache implementation supports -// automatic prereads; you can setup parameters on how this should be -// done using the next the next structure and method. The following -// options can be specified: -// -static const int SingleUse = 0x0001; // Mark pages for single use +//----------------------------------------------------------------------------- +//! Set automatic preread parameters for this file (may be ignored). +//! +//! @param aprP Reference to preread parameters. +//----------------------------------------------------------------------------- -virtual -void Preread (long long Offset, int Length, int Opts=0) -{ - (void)Offset; (void)Length; (void)Opts; -} - -// The following structure describes automatic preread parameters. These can be -// set at any time for each XrdOucCacheIO object. It can also be specified when -// creating a cache to establish the defaults (see XrdOucCache::Create()). -// Generally, an implementation that supports prereads should disable small -// prereads when minPages or loBound is set to zero; and should disable large -// prereads when maxiRead or maxPages is set to zero. Refer to the actual -// derived class implementation on how the cache handles prereads. -// struct aprParms {int Trigger; // preread if (rdln < Trigger) (0 -> pagesize+1) int prRecalc; // Recalc pr efficiency every prRecalc bytes (0->50M) - int Reserve4; + int Reserve1; short minPages; // If rdln/pgsz < min, preread minPages (0->off) signed char minPerf; // Minimum auto preread performance required (0->n/a) - char Reserve1; + char Reserve2; + void *Reserve3; - aprParms() : Trigger(0), prRecalc(0), Reserve4(0), - minPages(0), minPerf(90), Reserve1(0) - {} + aprParms() : Trigger(0), prRecalc(0), Reserve1(0), + minPages(0), minPerf(90), Reserve2(0), Reserve3(0) {} }; -virtual -void Preread(aprParms &Parms) { (void)Parms; } +virtual void Preread(aprParms &Parms) { (void)Parms; } + +//------------------------------------------------------------------------------ +//! Perform an asynchronous read (defaults to synchronous). +//! +//! @param iocb reference to the callback object that receives the result. All +//! results are returned via this object's Done() method. If the +//! caller holds any locks they must be recursive locks as the +//! callback may occur on the calling thread. +//! @param buff pointer to the buffer to receive the results. The buffer must +//! remain valid until the callback is invoked. +//! @param offs the offset into the file. +//! @param rlen the number of bytes to read. +//! +//! @return < 0 - Read failed, value is -errno. +//! >=0 - Read succeeded, value is number of bytes read. +//------------------------------------------------------------------------------ + +virtual int Read (char *buff, long long offs, int rlen) = 0; + +virtual void Read (XrdOucCacheIOCB &iocb, char *buff, long long offs, int rlen) + {iocb.Done(Read(buff, offs, rlen));} + +//------------------------------------------------------------------------------ +//! Perform an asynchronous vector read (defaults to synchronous). +//! +//! @param iocb reference to the callback object that receives the result. All +//! results are returned via this object's Done() method. If the +//! caller holds any locks they must be recursive locks as the +//! callback may occur on the calling thread. +//! @param readV pointer to a vector of read requests. +//! @param rnum the number of elements in the vector. +//! +//! @return < 0 - ReadV failed, value is -errno. +//! >=0 - ReadV succeeded, value is number of bytes read. +//------------------------------------------------------------------------------ + +virtual int ReadV(const XrdOucIOVec *readV, int rnum) + {int nbytes = 0, curCount = 0; + for (int i = 0; i < rnum; i++) + {curCount = Read(readV[i].data, + readV[i].offset, + readV[i].size); + if (curCount != readV[i].size) + return (curCount < 0 ? curCount : -ESPIPE); + nbytes += curCount; + } + return nbytes; + } + +virtual void ReadV(XrdOucCacheIOCB &iocb, const XrdOucIOVec *readV, int rnum) + {iocb.Done(ReadV(readV, rnum));} + +//------------------------------------------------------------------------------ +//! Perform an asynchronous sync() operation (defaults to synchronous). +//! +//! @param iocb reference to the callback object that receives the result. All +//! results are returned via this object's Done() method. If the +//! caller holds any locks they must be recursive locks as the +//! callback may occur on the calling thread. +//! +//! @return <0 - Sync failed, value is -errno. +//! =0 - Sync succeeded. +//------------------------------------------------------------------------------ + +virtual int Sync() = 0; + +virtual void Sync(XrdOucCacheIOCB &iocb) {iocb.Done(Sync());} + +//------------------------------------------------------------------------------ +//! Perform an asynchronous trunc() operation (defaults to synchronous). +//! +//! @param iocb reference to the callback object that receives the result. All +//! results are returned via this object's Done() method. If the +//! caller holds any locks they must be recursive locks as the +//! callback may occur on the calling thread. +//! @param offs the size the file is have. +//! +//! @return <0 - Trunc failed, value is -errno. +//! =0 - Trunc succeeded. +//------------------------------------------------------------------------------ + +virtual int Trunc(long long offs) = 0; + +virtual void Trunc(XrdOucCacheIOCB &iocb, long long offs) + {iocb.Done(Trunc(offs));} + +//------------------------------------------------------------------------------ +//! Update the originally passed XrdOucCacheIO object with the object passed. +//! All future uses underlying XrdOucCacheIO object must now use this object. +//! Update() is called when Prepare() indicated that the file should not be +//! physically opened and a file method was invoked in the XrdOucCacheIO +//! passed to Attach(). When this occurs, the file is actually opened and +//! Update() called to replace the original XrdOucCacheIO object with one +//! that uses the newly opened file. +//! +//! @param iocp reference to the new XrdOucCacheIO object. +//------------------------------------------------------------------------------ + +virtual void Update(XrdOucCacheIO &iocp) {} + +//------------------------------------------------------------------------------ +//! Perform an asynchronous write (defaults to synchronous). +//! +//! @param iocb reference to the callback object that receives the result. All +//! results are returned via this object's Done() method. If the +//! caller holds any locks they must be recursive locks as the +//! callback may occur on the calling thread. +//! @param buff pointer to the buffer holding the contents. The buffer must +//! remain valid until the callback is invoked. +//! @param offs the offset into the file. +//! @param wlen the number of bytes to write +//! +//! @return < 0 - Write failed, value is -errno. +//! >=0 - Write succeeded, value is number of bytes written. +//------------------------------------------------------------------------------ + +virtual int Write(char *buff, long long offs, int wlen) = 0; + +virtual void Write(XrdOucCacheIOCB &iocb, char *buff, long long offs, int wlen) + {iocb.Done(Write(buff, offs, wlen));} + +//------------------------------------------------------------------------------ +// Statistics about individual file usage reside here. There is also an overall +// summary CacheStats object in the cache object itself. +//------------------------------------------------------------------------------ -// Here is where the stats about cache and I/O usage reside. There -// is a summary object in the associated cache as well. -// XrdOucCacheStats Statistics; +//------------------------------------------------------------------------------ +//! Construct and Destructor +//------------------------------------------------------------------------------ + + XrdOucCacheIO() {} +protected: virtual ~XrdOucCacheIO() {} // Always use Detach() instead of direct delete! }; @@ -273,121 +337,204 @@ virtual ~XrdOucCacheIO() {} // Always use Detach() instead of direct delete! /* C l a s s X r d O u c C a c h e */ /******************************************************************************/ -/* The XrdOucCache class is used to define an instance of a cache. There can - be many such instances. Each instance is associated with one or more - XrdOucCacheIO objects. Use the Attach() method in this class to create - such associations. -*/ - -class XrdOucEnv; +//------------------------------------------------------------------------------ +//! The XrdOucCache class is used to define a cache. The cache is associated +//! with one or more XrdOucCacheIO objects using the Attach() method. +//------------------------------------------------------------------------------ class XrdOucCache { public: -/* Attach() must be called to obtain a new XrdOucCacheIO object that fronts an - existing XrdOucCacheIO object with this cache. - Upon success a pointer to a new XrdOucCacheIO object is returned - and must be used to read and write data with the cache interposed. - Upon failure, the original XrdOucCacheIO object is returned with - errno set. You can continue using the object without any cache. - The following Attach() options are available and, when specified, - override the default options associated with the cache, except for - optRW, optNEW, and optWIN which are valid only for a r/w cache. -*/ -static const int optADB = 0x1000; // Automatically delete underlying CacheIO -static const int optFIS = 0x0001; // File is Structured (e.g. root file) -static const int optFIU = 0x0002; // File is Unstructured (e.g. unix file) -static const int optRW = 0x0004; // File is read/write (o/w read/only) -static const int optNEW = 0x0014; // File is new -> optRW (o/w read to write) -static const int optWIN = 0x0024; // File is new -> optRW use write-in cache +//------------------------------------------------------------------------------ +//! Obtain a new XrdOucCacheIO object that fronts an existing XrdOucCacheIO +//! with this cache. Upon success a pointer to a new XrdOucCacheIO object is +//! returned and must be used to read and write data with the cache interposed. +//! Upon failure, the original XrdOucCacheIO object is returned with errno set. +//! You can continue using the object without any cache. The new cache should +//! use the methods in the passed CacheIO object to perform I/O operatios. +//! +//! @param ioP Pointer to the current CacheIO object used for I/O. +//! @param opts Cache options as defined below. When specified, they +//! override the default options associated with the cache, +//! except for optRW, optNEW, and optWIN which are valid only +//! for a r/w cache. +//! +//! @return Pointer to a new XrdOucCacheIO object (success) or the original +//! XrdOucCacheIO object (failure) with errno set. +//------------------------------------------------------------------------------ -virtual -XrdOucCacheIO *Attach(XrdOucCacheIO *ioP, int Options=0) = 0; +static const int optFIS = 0x0001; //!< File is structured (e.g. root file) +static const int optRW = 0x0004; //!< File is read/write (o/w read/only) +static const int optNEW = 0x0014; //!< File is new -> optRW (o/w read or write) +static const int optWIN = 0x0024; //!< File is new -> optRW use write-in cache -/* isAttached() - Returns the number of CacheIO objects attached to this cache. - Hence, 0 (false) if none and true otherwise. -*/ virtual -int isAttached() {return 0;} - -/* You must first create an instance of a cache using the Create() method. - The Parms structure is used to pass parameters about the cache and should - be filled in with values meaningful to the type of cache being created. - The fields below, while oriented toward a memory cache, are sufficiently - generic to apply to almost any kind of cache. Refer to the actual - implementation in the derived class to see how these values are used. -*/ -struct Parms - {long long CacheSize; // Size of cache in bytes (default 100MB) - int PageSize; // Size of each page in bytes (default 32KB) - int Max2Cache; // Largest read to cache (default PageSize) - int MaxFiles; // Maximum number of files (default 256 or 8K) - int Options; // Options as defined below (default r/o cache) - short minPages; // Minum number of pages (default 256) - short Reserve1; // Reserved for future use - int Reserve2; // Reserved for future use - - Parms() : CacheSize(104857600), PageSize(32768), - Max2Cache(0), MaxFiles(0), Options(0), - minPages(0), Reserve1(0), Reserve2(0) {} - }; +XrdOucCacheIO *Attach(XrdOucCacheIO *ioP, int opts=0) = 0; -// Valid option values in Parms::Options -// -static const int -isServer = 0x0010; // This is server application (as opposed to a user app). - // Appropriate internal optimizations will be used. -static const int -isStructured = 0x0020; // Optimize for structured files (e.g. root). +//------------------------------------------------------------------------------ +//! Get the path to a file that is complete in the local cache. By default, the +//! file must be complete in the cache (i.e. no blocks are missing). This can +//! be overridden. Thes path can be used to access the file on the local node. +//! +//! @param url - Pointer to the url of interest. +//! @param buff - Pointer to a buffer to receive the local path to the file. +//! If nil, no path is returned. +//! @param blen - Length of the buffer, buff. If zero, no path is returned. +//! @param why - One of the LFP_Reason enums describing the call: +//! ForAccess - the path will be used to access the file. If +//! the file is complete, the system will delay +//! purging the file for a configurable window, +//! should a purge be imminent. A null path is +//! returned for any non-zero return code. +//! ForInfo - same as ForAccess except that purging will +//! not be delayed if imminent. A path is always +//! returned, if possible. Otherwise the first +//! byte of any supplied buffer is set to 0. +//! ForPath - Only the path is wanted and no checks need +//! be performed. The only possible errors are +//! -EINVAL and -ENAMETOOLONG. +//! +//! @return 0 - the file is complete and the local path to the file is in +//! the buffer, if it has been supllied. +//! +//! @return <0 - the request could not be fulfilled. The return value is +//! -errno describing why. If a buffer was supplied and a +//! path could be generated it is returned only if "why" is +//! ForInfo or ForPath. Otherwise, a null path is returned. +//! +//! Common return codes are: +//! -EINVAL an argument is invalid. +//! -EISDIR target is a directory not a file. +//! -ENAMETOOLONG buffer not big enough to hold path. +//! -ENOENT file not in cache +//! -ENOTSUP method not implemented +//! -EREMOTE file is incomplete +//! +//! @return >0 - Reserved for future use. +//------------------------------------------------------------------------------ -static const int -canPreRead = 0x0040; // Enable pre-read operations (o/w ignored) +enum LFP_Reason {ForAccess=0, ForInfo, ForPath}; -static const int -logStats = 0x0080; // Display statistics upon detach +virtual int LocalFilePath(const char *url, char *buff=0, int blen=0, + LFP_Reason why=ForAccess) + {(void)url; (void)buff; (void)blen; (void)why; + if (buff && blen > 0) *buff = 0; + return -ENOTSUP; + } -static const int -Serialized = 0x0004; // Caller ensures MRSW semantics +//------------------------------------------------------------------------------ +//! Prepare the cache for a file open request. This method is called prior to +//! actually opening a file. This method is meant to allow defering an open +//! request or implementing the full I/O stack in the cache layer. +//! +//! @param url - Pointer to the url about to be opened. +//! @param oflags - Standard Unix open flags (see open(2)). +//! @param mode - Standard mode flags if file is being created. +//! +//! @return <0 Error has occurred, return value is -errno; fail open request. +//! The error code -EUSERS may be returned to trigger overload +//! recovery as specified by the xrootd.fsoverload directive. No +//! other method should return this error code. +//! =0 Continue with open() request. +//! >0 Defer open but treat the file as actually being open. +//------------------------------------------------------------------------------ -static const int -ioMTSafe = 0x0008; // CacheIO object is MT-safe +virtual int Prepare(const char *url, int oflags, mode_t mode) + {(void)url; (void)oflags; (void)mode; return 0;} -static const int -Debug = 0x0003; // Produce some debug messages (levels 0, 1, 2, or 3) +//------------------------------------------------------------------------------ +//! Rename a file in the cache. +//! +//! @param oldp - the existing path to be renamed. +//! @param newp - the new name it is to have. +//! +//! @return Success: 0 +//! @return Failure: -errno +//------------------------------------------------------------------------------ -/* Create() Creates an instance of a cache using the specified parameters. - You must pass the cache parms and optionally any automatic - pre-read parameters that will be used as future defaults. - Upon success, returns a pointer to the cache. Otherwise, a null - pointer is returned with errno set to indicate the problem. -*/ -virtual -XrdOucCache *Create(Parms &Params, XrdOucCacheIO::aprParms *aprP=0) = 0; +virtual int Rename(const char* oldp, const char* newp) + {(void)oldp; (void)newp; return 0;} +//------------------------------------------------------------------------------ +//! Remove a directory from the cache. +//! +//! @param dirp - the existing directory path to be removed. +//! +//! @return Success: 0 +//! @return Failure: -errno +//------------------------------------------------------------------------------ -// Propagate Unlink client request from posix layer to cache. -virtual -int Unlink(const char* /*path*/) { return 0; } +virtual int Rmdir(const char* dirp) {(void)dirp; return 0;} -// Propagate Rmdir client request from posix layer to cache. -virtual -int Rmdir(const char* /*path*/) { return 0; } +//------------------------------------------------------------------------------ +//! Perform a stat() operation (defaults to passthrough). +//! +//! @param url pointer to the url whose stat information is wanted. +//! @param sbuff reference to the stat buffer to be filled in. Only fields +//! st_size, st_blocks, st_mtime (st_atime and st_ctime may be +//! set to st_mtime), st_ino, and st_mode need to be set. All +//! other fields are preset and should not be changed. +//! +//! @return <0 - Stat failed, value is -errno. +//! =0 - Stat succeeded, sbuff holds stat information. +//! >0 - Stat could not be done, forward operation to next level. +//------------------------------------------------------------------------------ -// Propagate Rename client request from posix layer to cache. -virtual -int Rename(const char* /*path*/, const char* /*newPath*/) { return 0; } +virtual int Stat(const char *url, struct stat &sbuff) + {(void)url; (void)sbuff; return 1;} -// Propagate Truncate client request from posix layer to cache. -virtual -int Truncate(const char* /*path*/, off_t /*size*/) { return 0; } +//------------------------------------------------------------------------------ +//! Truncate a file in the cache to a specified size. +//! +//! @param path - the path of the file to truncate. +//! @param size - the size in bytes it is to have. +//! +//! @return Success: 0 +//! @return Failure: -errno +//------------------------------------------------------------------------------ + +virtual int Truncate(const char* path, off_t size) + {(void)path; (void)size; return 0;} + +//------------------------------------------------------------------------------ +//! Remove a file from the cache. +//! +//! @param path - the path of the file to be removed. +//! +//! @return Success: 0 +//! @return Failure: -errno +//------------------------------------------------------------------------------ + +virtual int Unlink(const char* path) {(void)path; return 0;} + +//------------------------------------------------------------------------------ +//! Perform special operation on the cache. +//! +//! @param cmd - The operation to be performed. +//! @param arg - The operation argument(s). +//! @param arglen - The length of arg. +//! +//! @return Success: 0 +//! @return Failure: -errno +//------------------------------------------------------------------------------ + +enum XeqCmd {xeqNoop = 0}; + +virtual int Xeq(XeqCmd cmd, char *arg, int arglen) + {(void)cmd; (void)arg; (void)arglen; return -ENOTSUP;} + +//------------------------------------------------------------------------------ +//! The following holds statistics for the cache itself. It is updated as +//! associated cacheIO objects are deleted and their statistics are added. +//------------------------------------------------------------------------------ -/* The following holds statistics for the cache itself. It is updated as - associated cacheIO objects are deleted and their statistics are added. -*/ XrdOucCacheStats Stats; +//------------------------------------------------------------------------------ +//! Constructor & Destructor +//------------------------------------------------------------------------------ + XrdOucCache() {} virtual ~XrdOucCache() {} }; @@ -396,33 +543,47 @@ virtual ~XrdOucCache() {} /* C r e a t i n g C a c h e P l u g - I n s */ /******************************************************************************/ -/* You can create a cache plug-in for those parts of the xrootd system that - allow a dynamically selectable cache implementation (e.g. the proxy server - plug-in supports cache plug-ins via the pss.cachelib directive). - - Your plug-in must exist in a shared library and have the following extern C - function defined: - - extern "C" - { - XrdOucCache *XrdOucGetCache(XrdSysLogger *Logger, // Where messages go - const char *Config, // Config file used - const char *Parms, // Optional parm string - XrdOucEnv *envP); // Optional environment - } - - When Logger is null, you should use cerr to output messages. Otherwise, - tie an instance XrdSysError to the passed logger. - When Config is null, no configuration file is present. Otherwise, you need - additional configuration information you should get it - from that file in order to support single configuration. - When Parms is null, no parameter string was specified. - - The call should return an instance of an XrdOucCache object upon success and - a null pointer otherwise. The instance is used to create actual caches using - the object's Create() method. -*/ +//------------------------------------------------------------------------------ +//! Your cache plug-in must exist in a shared library and have the following +//! extern C function defined whos parameters are: +//! +//! @param Logger Pointer to the logger object that should be used with an +//! instance of XrdSysError to direct messages to a log file. +//! If Logger is null, you should use cerr to output messages. +//! @param Config Pointer to the configuration file name from where you +//! should get additional information. If Config is null, there +//! is no configuration file is present. +//! @param Parms Pointer to any parameters specified after the shared library +//! path. If Parms is null, there are no parameters. +//! @param envP Pointer to environmental information. The most relevant +//! is whether or not gStream monitoring is enabled. +//! XrdXrootdGStream *gStream = (XrddXrootdGStream *) +//! envP->GetPtr("pfc.gStream*"); +//! @return A usable, fully configured, instance of an XrdOucCache +//! object upon success and a null pointer otherwise. This +//! instance is used for all operations defined by methods in +//! XrdOucCache base class. +//! +//! extern "C" +//! { +//! XrdOucCache *XrdOucGetCache(XrdSysLogger *Logger, // Where messages go +//! const char *Config, // Config file used +//! const char *Parms, // Optional parm string +//! } XrdOucEnv *envP); // Optional environment typedef XrdOucCache *(*XrdOucCache_t)(XrdSysLogger *, const char *, const char *, XrdOucEnv *); + +//------------------------------------------------------------------------------ +//! Declare compilation version. +//! +//! Additionally, you *should* declare the xrootd version you used to compile +//! your plug-in. Declare it as shown below. +//------------------------------------------------------------------------------ + +/*! #include "XrdVersion.hh" + XrdVERSIONINFO(XrdOucGetCache,); + + where is a 1- to 15-character unquoted name identifying your plugin. +*/ #endif diff --git a/src/XrdOuc/XrdOucCache2.hh b/src/XrdOuc/XrdOucCache2.hh deleted file mode 100644 index c37b4310c72..00000000000 --- a/src/XrdOuc/XrdOucCache2.hh +++ /dev/null @@ -1,390 +0,0 @@ -#ifndef __XRDOUCCACHE2_HH__ -#define __XRDOUCCACHE2_HH__ -/******************************************************************************/ -/* */ -/* X r d O u c C a c h e 2 . h h */ -/* */ -/* (c) 2016 by the Board of Trustees of the Leland Stanford, Jr., University */ -/* All Rights Reserved */ -/* Produced by Andrew Hanushevsky for Stanford University under contract */ -/* DE-AC02-76-SFO0515 with the Department of Energy */ -/* */ -/* This file is part of the XRootD software suite. */ -/* */ -/* XRootD is free software: you can redistribute it and/or modify it under */ -/* the terms of the GNU Lesser General Public License as published by the */ -/* Free Software Foundation, either version 3 of the License, or (at your */ -/* option) any later version. */ -/* */ -/* XRootD is distributed in the hope that it will be useful, but WITHOUT */ -/* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ -/* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ -/* License for more details. */ -/* */ -/* You should have received a copy of the GNU Lesser General Public License */ -/* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ -/* COPYING (GPL license). If not, see . */ -/* */ -/* The copyright holder's institutional names and contributor's names may not */ -/* be used to endorse or promote products derived from this software without */ -/* specific prior written permission of the institution or contributor. */ -/******************************************************************************/ - -#include - -#include "XrdOuc/XrdOucCache.hh" - -//----------------------------------------------------------------------------- -//! XrdOucCache2 -//! -//! This class is an extension of the original XrdOucCache class and provides -//! async I/O support, defered open and I/O decoupling. It is loaded as a -//! cache plugin by the POSIX package via the proxy directive -//! -//! pss.cache -2 \ -//! -//! Without the "-2" the original version is loaded. Implementation of a cache -//! is similar between versions (see XrdOucCache.hh for details). The only -//! difference is this version requires additional methods to be implemented -//! and uses an asynchrnous callback mechanism to return the results. -//----------------------------------------------------------------------------- - -/******************************************************************************/ -/* C l a s s X r d O u c C a c h e I O 2 */ -/******************************************************************************/ - -//------------------------------------------------------------------------------ -//! The XrdOucCacheIO2 object is responsible for interacting with the original -//! data source/target. It can be used with or without a front-end cache. It -//! an extension of the XrdOucCacheIO class which defines base methods. -//------------------------------------------------------------------------------ - -class XrdOucCacheIO2 : public virtual XrdOucCacheIO -{ -public: - -//------------------------------------------------------------------------------ -//! Perform an fstat() operation (defaults to passthrough). -//! -//! @param sbuff reference to the stat buffer to be filled in. Only fields -//! st_size, st_blocks, st_mtime (st_atime and st_ctime may be -//! set to st_mtime), st_ino, and st_mode need to be set. All -//! other fields are preset and should not be changed. -//! -//! @return <0 - fstat failed, value is -errno. -//! =0 - fstat succeeded, sbuff holds stat information. -//! >0 - fstat could not be done, forward operation to next level. -//------------------------------------------------------------------------------ - -virtual int Fstat(struct stat &sbuff) {(void)sbuff; return 1;} - -//----------------------------------------------------------------------------- -//! Get the file's location (i.e. endpoint hostname and port) -//! -//! @return A pointer to the file's location. It remains valid until the file -//! is closed. A null string means the file is not open or is unknown. -//----------------------------------------------------------------------------- -virtual -const char *Location() {return "";} - -//------------------------------------------------------------------------------ -//! Perform an asynchronous read (defaults to synchrnous). -//! -//! @param iocb reference to the callback object that receives the result. All -//! results are returned via this object's Done() method. If the -//! caller holds any locks they must be recursive locks as the -//! callback may occur on the calling thread. Done() is passed -//! < 0 - Read failed, value is -errno. -//! >=0 - Read succeeded, value is number of bytes read. -//! @param buff pointer to the buffer to receive the results. The buffer must -//! remain valid until the callback is invoked. -//! @param offs the offset into the file. -//! @param rlen the number of bytes to read. -//------------------------------------------------------------------------------ - -using XrdOucCacheIO::Read; - -virtual void Read (XrdOucCacheIOCB &iocb, char *buff, long long offs, int rlen) - {iocb.Done(Read(buff, offs, rlen));} - -//------------------------------------------------------------------------------ -//! Perform an asynchronous vector read (defaults to synchrnous). -//! -//! @param iocb reference to the callback object that receives the result. All -//! results are returned via this object's Done() method. If the -//! caller holds any locks they must be recursive locks as the -//! callback may occur on the calling thread. Done() is passed -//! < 0 - ReadV failed, value is -errno. -//! >=0 - ReadV succeeded, value is number of bytes read. -//! @param readV pointer to a vector of read requests. -//! @param rnum the number of elements in the vector. -//------------------------------------------------------------------------------ - -using XrdOucCacheIO::ReadV; - -virtual void ReadV(XrdOucCacheIOCB &iocb, const XrdOucIOVec *readV, int rnum) - {iocb.Done(ReadV(readV, rnum));} - -//------------------------------------------------------------------------------ -//! Perform an asynchronous fsync() operation (defaults to synchronous). -//! -//! @param iocb reference to the callback object that receives the result. All -//! results are returned via this object's Done() method. If the -//! caller holds any locks they must be recursive locks as the -//! callback may occur on the calling thread. Done() is passed -//! <0 - Sync failed, value is -errno. -//! =0 - Sync succeeded. -//------------------------------------------------------------------------------ - -using XrdOucCacheIO::Sync; - -virtual void Sync(XrdOucCacheIOCB &iocb) {iocb.Done(Sync());} - -//------------------------------------------------------------------------------ -//! Update the originally passed XrdOucCacheIO2 object with the object passed. -//! All future uses underlying XrdOucCacheIO2 object must now use this object. -//! Update() is called when Prepare() indicated that the file should not be -//! physically opened and a file method was invoked in the XrdOucCacheIO2 -//! passed to Attach(). When this occurs, the file is actually opened and -//! Update() called to replace the original XrdOucCacheIO2 object with one -//! that uses the newly opened file. -//! -//! @param iocp reference to the new XrdOucCacheIO2 object. -//------------------------------------------------------------------------------ - -virtual void Update(XrdOucCacheIO2 &iocp) {} - -//------------------------------------------------------------------------------ -//! Perform an asynchronous write (defaults to synchronous). -//! -//! @param iocb reference to the callback object that receives the result. All -//! results are returned via this object's Done() method. If the -//! caller holds any locks they must be recursive locks as the -//! callback may occur on the calling thread. -//! < 0 - Write failed, value is -errno. -//! >=0 - Write succeeded, value is number of bytes written. -//! @param buff pointer to the buffer holding the contents. The buffer must -//! remain valid until the callback is invoked. -//! @param offs the offset into the file. -//! @param wlen the number of bytes to write -//------------------------------------------------------------------------------ - -using XrdOucCacheIO::Write; - -virtual void Write(XrdOucCacheIOCB &iocb, char *buff, long long offs, int wlen) - {iocb.Done(Write(buff, offs, wlen));} - -//------------------------------------------------------------------------------ - -virtual ~XrdOucCacheIO2() {} // Always use Detach() instead of direct delete! -}; - -/******************************************************************************/ -/* C l a s s X r d O u c C a c h e 2 */ -/******************************************************************************/ - -class XrdOucEnv; -struct stat; - -//------------------------------------------------------------------------------ -//! The XrdOucCache2 class is used to define a version 2 cache. In version 2, -//! there can be only one such instance. the cache is associated with one or -//! more XrdOucCacheIO2 objects. Use the Attach() method in this class to -//! create such associations. -//------------------------------------------------------------------------------ - -class XrdOucCache2 : public virtual XrdOucCache -{ -public: - -//------------------------------------------------------------------------------ -//! Obtain a new XrdOucCacheIO2 object that fronts an existing XrdOucCacheIO2 -//! with this cache. Upon success a pointer to a new XrdOucCacheIO2 object is -//! returned and must be used to read and write data with the cache interposed. -//! Upon failure, the original XrdOucCacheIO2 object is returned with errno set. -//! You can continue using the object without any cache. The new cache should -//! use the methods in the passed CacheIO2 object to perform I/O operatios. -//! -//! @param ioP Pointer to the current CacheIO2 object used for I/O. -//! @param opts Cache options identical to those defined for XrdOucCache -//! Attach() method. -//! -//! @return Pointer to a new XrdOucCacheIO2 object (success) or the original -//! XrdOucCacheIO2 object (failure) with errno set. -//------------------------------------------------------------------------------ - -using XrdOucCache::Attach; - -virtual -XrdOucCacheIO2 *Attach(XrdOucCacheIO2 *ioP, int opts=0) = 0; - -virtual -XrdOucCacheIO *Attach(XrdOucCacheIO *ioP, int opts=0) - {errno = ENOSYS; return ioP;} - -//------------------------------------------------------------------------------ -//! Creates an instance of a version 1 cache. This method is no longer used so -//! we simply define a default for this method here for backward compatability. -//! -//! @return A pointer to an XrdOucCache object upon success or a nil pointer -//! with errno set upon failure. -//------------------------------------------------------------------------------ -virtual -XrdOucCache *Create(Parms &Params, XrdOucCacheIO::aprParms *aprP=0) - {return this;} - -//------------------------------------------------------------------------------ -//! Supply environmental information to the cache. This is only called on the -//! server but is optional and might not be called. When it is called, it is -//! gauranteed to occur before any active use of the cache and is essentially -//! serialized (i.e. the main start-up thread is used). The environmental -//! information should only be used to optimize processing. For instance, -//! when cache monitoring is enabled, the variable "pfc.gStream*" is defined -//! and is a pointer to a gStream object that can be used to report statistical -//! information to a monitoring collector. -//! -//! @param theEnv - Reference to environmental information. -//------------------------------------------------------------------------------ -virtual -void EnvInfo(XrdOucEnv &theEnv) {(void)theEnv;} - -//------------------------------------------------------------------------------ -//! Get the path to a file that is complete in the local cache. By default, the -//! file must be complete in the cache (i.e. no blocks are missing). This can -//! be overridden. Thes path can be used to access the file on the local node. -//! -//! @param url - Pointer to the url of interest. -//! @param buff - Pointer to a buffer to receive the local path to the file. -//! If nil, no path is returned. -//! @param blen - Length of the buffer, buff. If zero, no path is returned. -//! @param why - One of the LFP_Reason enums describing the call: -//! ForAccess - the path will be used to access the file. If -//! the file is complete, the system will delay -//! purging the file for a configurable window, -//! should a purge be imminent. A null path is -//! returned for any non-zero return code. -//! ForInfo - same as ForAccess except that purging will -//! not be delayed if imminent. A path is always -//! returned, if possible. Otherwise the first -//! byte of any supplied buffer is set to 0. -//! ForPath - Only the path is wanted and no checks need -//! be performed. The only possible errors are -//! -EINVAL and -ENAMETOOLONG. -//! @param forall - When ForAccess is specified: when true makes the file -//! world readable; otherwise, only group readable. The -//! parameter is ignored unless "why" is ForAccess and a -//! local file path is requested to be returned. -//! -//! @return 0 - the file is complete and the local path to the file is in -//! the buffer, if it has been supllied. -//! -//! @return <0 - the request could not be fulfilled. The return value is -//! -errno describing why. If a buffer was supplied and a -//! path could be generated it is returned only if "why" is -//! ForInfo or ForPath. Otherwise, a null path is returned. -//! -//! Common return codes are: -//! -EINVAL an argument is invalid. -//! -EISDIR target is a directory not a file. -//! -ENAMETOOLONG buffer not big enough to hold path. -//! -ENOENT file not in cache -//! -ENOTSUP method not implemented -//! -EREMOTE file is incomplete -//! -//! @return >0 - Reserved for future use. -//------------------------------------------------------------------------------ - -enum LFP_Reason {ForAccess=0, ForInfo, ForPath}; - -virtual -int LocalFilePath(const char *url, char *buff=0, int blen=0, - LFP_Reason why=ForAccess, bool forall=false) - {(void)url; (void)buff; (void)blen; (void)why; - if (buff && blen > 0) *buff = 0; - return -ENOTSUP; - } - -//------------------------------------------------------------------------------ -//! Prepare the cache for a file open request. This method is called prior to -//! actually opening a file. This method is meant to allow defering an open -//! request or implementing the full I/O stack in the cache layer. -//! -//! @param url - Pointer to the url about to be opened. -//! @param oflags - Standard Unix open flags (see open(2)). -//! @param mode - Standard mode flags if file is being created. -//! -//! @return <0 Error has occurred, return value is -errno; fail open request. -//! The error code -EUSERS may be returned to trigger overload -//! recovery as specified by the xrootd.fsoverload directive. No -//! other method should return this error code. -//! =0 Continue with open() request. -//! >0 Defer open but treat the file as actually being open. Use the -//! XrdOucCacheIO2::Open() method to open the file at a later time. -//------------------------------------------------------------------------------ -virtual -int Prepare(const char *url, int oflags, mode_t mode) - {(void)url; (void)oflags; (void)mode; return 0;} - -//------------------------------------------------------------------------------ -//! Perform a stat() operation (defaults to passthrough). -//! -//! @param url pointer to the url whose stat information is wanted. -//! @param sbuff reference to the stat buffer to be filled in. Only fields -//! st_size, st_blocks, st_mtime (st_atime and st_ctime may be -//! set to st_mtime), st_ino, and st_mode need to be set. All -//! other fields are preset and should not be changed. -//! -//! @return <0 - Stat failed, value is -errno. -//! =0 - Stat succeeded, sbuff holds stat information. -//! >0 - Stat could not be done, forward operation to next level. -//------------------------------------------------------------------------------ - -virtual int Stat(const char *url, struct stat &sbuff) - {(void)url; (void)sbuff; return 1;} - - XrdOucCache2() {} -virtual ~XrdOucCache2() {} -}; - -/******************************************************************************/ -/* C r e a t i n g C a c h e P l u g - I n s */ -/******************************************************************************/ - -//------------------------------------------------------------------------------ -//! You can create a V2 cache plug-in for those parts of the xrootd system that -//! allow a dynamically selectable cache implementation (e.g. the proxy server -//! plug-in supports cache plug-ins via the pss.cachelib). For the proxy server, -//! it preferentially looks for XrdOucGetCache2() and invokes this function if -//! is exists. Otherwise, it uses the XrdOucGetCache() function (old version 1). -//! -//! Your plug-in must exist in a shared library and have the following extern C -//! function defined whos parameters are: -//! -//! @param Logger Pointer to the logger object that should be used with an -//! instance of XrdSysError to direct messages to a log file. -//! If Logger is null, you should use cerr to output messages. -//! @param Config Pointer to the configuration file name from where you -//! should get additional information. If Config is null, there -//! is no configuration file is present. -//! @param Parms Pointer to any parameters specified after the shared library -//! path. If Parms is null, there are no parameters. -//! @param envP Pointer to environmental information. The most relevant -//! is whether or not -stream monitoring is enabled. -//! XrdXrootdGStream *gstm = envP->(XrddXrootdGStream *) -//! GetPtr("pfc.gStream*"); -//! @return A usable, fully configured, instance of an XrdOucCache2 -//! object upon success and a null pointer otherwise. This -//! instance is used for all operations defined by methods in -//! XrdOucCache bas class as well as this class. -//! -//! extern "C" -//! { -//! XrdOucCache2 *XrdOucGetCache2(XrdSysLogger *Logger, // Where messages go -//! const char *Config, // Config file used -//! const char *Parms, // Optional parm string -//! } XrdOucEnv *envP); // Optional environment - -typedef XrdOucCache2 *(*XrdOucCache2_t)(XrdSysLogger *, const char *, - const char *, XrdOucEnv *); - -#endif diff --git a/src/XrdOuc/XrdOucPsx.cc b/src/XrdOuc/XrdOucPsx.cc index 4bed88f9024..174c531b8bf 100644 --- a/src/XrdOuc/XrdOucPsx.cc +++ b/src/XrdOuc/XrdOucPsx.cc @@ -37,7 +37,7 @@ #include #include "XrdOuc/XrdOuca2x.hh" -#include "XrdOuc/XrdOucCache2.hh" +#include "XrdOuc/XrdOucCache.hh" #include "XrdOuc/XrdOucEnv.hh" #include "XrdOuc/XrdOucN2NLoader.hh" #include "XrdOuc/XrdOucName2Name.hh" @@ -171,30 +171,13 @@ bool XrdOucPsx::ClientConfig(const char *pfx, bool hush) bool XrdOucPsx::ConfigCache(XrdSysError &eDest) { XrdOucPinLoader myLib(&eDest,myVersion,"cachelib",cPath); - const char *cName; - bool isCache2; -// First find out if this is a cache2 or old cache library +// Get the cache Object now // - if (myLib.Resolve("?XrdOucGetCache2") == 0) - {cName = "XrdOucGetCache"; isCache2 = false; - } else { - cName = "XrdOucGetCache2"; isCache2 = true; - } - -// Get the Object now -// - if (isCache2) - {XrdOucCache2_t ep = (XrdOucCache2_t)myLib.Resolve(cName); - if (!ep) return false; - theCache2 = (XrdOucCache2*)ep(eDest.logger(), configFN, cParm, theEnv); - return theCache2 != 0; - } else { - XrdOucCache_t ep = (XrdOucCache_t)myLib.Resolve(cName); - if (!ep) return false; - theCache = (XrdOucCache*)ep(eDest.logger(), configFN, cParm, theEnv); - return theCache != 0; - } + XrdOucCache_t ep = (XrdOucCache_t)myLib.Resolve("XrdOucGetCache"); + if (!ep) return false; + theCache = (XrdOucCache*)ep(eDest.logger(), configFN, cParm, theEnv); + return theCache != 0; } /******************************************************************************/ @@ -247,7 +230,7 @@ bool XrdOucPsx::ConfigSetup(XrdSysError &eDest, bool hush) eDest.logger()->Capture(&tFifo); } } else { - if (mPath && theCache2 && !LoadCCM(eDest)) + if (mPath && theCache && !LoadCCM(eDest)) {aOK = false; if (hush) {eDest.logger()->Capture(0); diff --git a/src/XrdOuc/XrdOucPsx.hh b/src/XrdOuc/XrdOucPsx.hh index 72bc7caca33..46fc11c765b 100644 --- a/src/XrdOuc/XrdOucPsx.hh +++ b/src/XrdOuc/XrdOucPsx.hh @@ -36,7 +36,6 @@ #include "XrdOuc/XrdOucCacheCM.hh" -class XrdOucCacheIF; class XrdOucEnv; class XrdOucName2Name; class XrdSysError; @@ -78,8 +77,7 @@ char *configFN; // -> Pointer to the config file name XrdSysLogger *theLogger; XrdOucEnv *theEnv; XrdOucName2Name *theN2N; // -> File mapper object -XrdOucCache *theCache; // -> V1 cache -XrdOucCache2 *theCache2; // -> V2 cache +XrdOucCache *theCache; // -> Cache object XrdOucCacheCMInit_t initCCM; // -> Cache context manager initializer char *mCache; // -> memory cache parameters XrdOucTList *setFirst; @@ -97,7 +95,7 @@ bool xNameLib; XrdOucPsx(XrdVersionInfo *vInfo, const char *cfn, XrdSysLogger *lp=0, XrdOucEnv *vp=0) : configFN(strdup(cfn)), theLogger(lp), theEnv(vp), - theN2N(0), theCache(0), theCache2(0), initCCM(0), + theN2N(0), theCache(0), initCCM(0), mCache(0), setFirst(0), setLast(0), maxRHCB(0), traceLvl(0), debugLvl(0), cioWait(0), cioTries(0), useV4(false), xLfn2Pfn(false), xPfn2Lfn(false), diff --git a/src/XrdPosix.cmake b/src/XrdPosix.cmake index f2f0e70af78..49dc565f7c7 100644 --- a/src/XrdPosix.cmake +++ b/src/XrdPosix.cmake @@ -17,7 +17,6 @@ add_library( SHARED XrdPosix/XrdPosixAdmin.cc XrdPosix/XrdPosixAdmin.hh XrdPosix/XrdPosixCache.cc XrdPosix/XrdPosixCache.hh - XrdPosix/XrdPosixCacheBC.hh XrdPosix/XrdPosixCallBack.cc XrdPosix/XrdPosixCallBack.hh XrdPosix/XrdPosixConfig.cc XrdPosix/XrdPosixConfig.hh XrdPosix/XrdPosixDir.cc XrdPosix/XrdPosixDir.hh diff --git a/src/XrdPosix/XrdPosixCache.cc b/src/XrdPosix/XrdPosixCache.cc index 79fc792e0e8..f18e25ff321 100644 --- a/src/XrdPosix/XrdPosixCache.cc +++ b/src/XrdPosix/XrdPosixCache.cc @@ -28,7 +28,7 @@ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ -#include "XrdOuc/XrdOucCache2.hh" +#include "XrdOuc/XrdOucCache.hh" #include "XrdOuc/XrdOucCacheStats.hh" #include "XrdPosix/XrdPosixCache.hh" @@ -38,7 +38,7 @@ namespace XrdPosixGlobals { -extern XrdOucCache2 *theCache; +extern XrdOucCache *theCache; } using namespace XrdPosixGlobals; @@ -49,7 +49,7 @@ using namespace XrdPosixGlobals; int XrdPosixCache::CachePath(const char *url, char *buff, int blen) { - return theCache->LocalFilePath(url, buff, blen, XrdOucCache2::ForPath); + return theCache->LocalFilePath(url, buff, blen, XrdOucCache::ForPath); } /******************************************************************************/ @@ -60,8 +60,8 @@ int XrdPosixCache::CacheQuery(const char *url, bool hold) { int rc = theCache->LocalFilePath(url, 0, 0, - (hold ? XrdOucCache2::ForAccess - : XrdOucCache2::ForInfo) + (hold ? XrdOucCache::ForAccess + : XrdOucCache::ForInfo) ); if (!rc) return 1; if (rc == -EREMOTE) return 0; diff --git a/src/XrdPosix/XrdPosixCacheBC.hh b/src/XrdPosix/XrdPosixCacheBC.hh deleted file mode 100644 index da13f825abc..00000000000 --- a/src/XrdPosix/XrdPosixCacheBC.hh +++ /dev/null @@ -1,136 +0,0 @@ -#ifndef __XRDPOSIXCACHEBC_HH__ -#define __XRDPOSIXCACHEBC_HH__ -/******************************************************************************/ -/* */ -/* X r d P o s i x C a c h e B C . h h */ -/* */ -/* (c) 2016 by the Board of Trustees of the Leland Stanford, Jr., University */ -/* All Rights Reserved */ -/* Produced by Andrew Hanushevsky for Stanford University under contract */ -/* DE-AC02-76-SFO0515 with the Department of Energy */ -/* */ -/* This file is part of the XRootD software suite. */ -/* */ -/* XRootD is free software: you can redistribute it and/or modify it under */ -/* the terms of the GNU Lesser General Public License as published by the */ -/* Free Software Foundation, either version 3 of the License, or (at your */ -/* option) any later version. */ -/* */ -/* XRootD is distributed in the hope that it will be useful, but WITHOUT */ -/* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ -/* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ -/* License for more details. */ -/* */ -/* You should have received a copy of the GNU Lesser General Public License */ -/* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ -/* COPYING (GPL license). If not, see . */ -/* */ -/* The copyright holder's institutional names and contributor's names may not */ -/* be used to endorse or promote products derived from this software without */ -/* specific prior written permission of the institution or contributor. */ -/******************************************************************************/ - -#include "XrdOuc/XrdOucCache2.hh" - -/******************************************************************************/ -/* X r d P o s i x C a c h e B C I O */ -/******************************************************************************/ - -class XrdPosixCacheBCIO : public XrdOucCacheIO2 -{ -public: - -virtual -XrdOucCacheIO2 *Base() {return cacheIO2;} - -virtual -XrdOucCacheIO2 *Detach() {XrdOucCacheIO2 *theCIO = cacheIO2; - cacheIO1->Detach(); - delete this; - return theCIO; - } - -virtual -long long FSize() {return cacheIO1->FSize();} - -virtual int Fstat(struct stat &buf) {return cacheIO2->Fstat(buf);} - -virtual -const char *Location() {return cacheIO2->Location();} - -virtual -const char *Path() {return cacheIO1->Path();} - -using XrdOucCacheIO2::Read; - -virtual int Read (char *Buffer, long long Offset, int Length) - {return cacheIO1->Read(Buffer, Offset, Length);} - -using XrdOucCacheIO2::ReadV; - -virtual int ReadV(const XrdOucIOVec *readV, int n) - {return cacheIO1->ReadV(readV, n);} - -using XrdOucCacheIO2::Sync; - -virtual int Sync() {return cacheIO1->Sync();} - -using XrdOucCacheIO2::Trunc; - -virtual int Trunc(long long Offset) {return cacheIO1->Trunc(Offset);} - -using XrdOucCacheIO2::Write; - -virtual int Write(char *Buffer, long long Offset, int Length) - {return cacheIO1->Write(Buffer, Offset, Length);} - -virtual bool ioActive() { return cacheIO1->ioActive();} - -virtual void Preread (long long Offset, int Length, int Opts=0) - {return cacheIO1->Preread(Offset, Length, Opts);} - -virtual void Preread(aprParms &Parms) { cacheIO1->Preread(Parms);} - - XrdPosixCacheBCIO(XrdOucCacheIO *urCIO, XrdOucCacheIO2 *myCIO) - : cacheIO1(urCIO), cacheIO2(myCIO) {} -virtual ~XrdPosixCacheBCIO() {} - -private: -XrdOucCacheIO *cacheIO1; -XrdOucCacheIO2 *cacheIO2; -}; - -/******************************************************************************/ -/* X r d P o s i x C a c h e B C */ -/******************************************************************************/ - -class XrdPosixCacheBC : public XrdOucCache2 -{ -public: -using XrdOucCache2::Attach; - -virtual -XrdOucCacheIO2 *Attach(XrdOucCacheIO2 *ioP, int opts=0) - {XrdOucCacheIO *newIOP = v1Cache->Attach(ioP, opts); - if (newIOP == (XrdOucCacheIO *)ioP) return ioP; - return new XrdPosixCacheBCIO(newIOP, ioP); - } - -virtual int isAttached() {return v1Cache->isAttached();} - -virtual int Rmdir(const char* path) {return v1Cache->Rmdir(path);} - -virtual int Rename(const char* pathO, const char* pathN) - {return v1Cache->Rename(pathO, pathN);} - -virtual int Truncate(const char* path, off_t size) - {return v1Cache->Truncate(path, size);} - -virtual int Unlink(const char* path) {return v1Cache->Unlink(path);} - - XrdPosixCacheBC(XrdOucCache *cP) : v1Cache(cP) {} -virtual ~XrdPosixCacheBC() {} -private: -XrdOucCache *v1Cache; -}; -#endif diff --git a/src/XrdPosix/XrdPosixConfig.cc b/src/XrdPosix/XrdPosixConfig.cc index 8ce20cc680d..ef9ec9e0274 100644 --- a/src/XrdPosix/XrdPosixConfig.cc +++ b/src/XrdPosix/XrdPosixConfig.cc @@ -34,14 +34,12 @@ #include "XrdCl/XrdClDefaultEnv.hh" -#include "XrdOuc/XrdOucCache2.hh" -#include "XrdOuc/XrdOucCacheDram.hh" +#include "XrdOuc/XrdOucCache.hh" #include "XrdOuc/XrdOucEnv.hh" #include "XrdOuc/XrdOucPsx.hh" #include "XrdOuc/XrdOucTList.hh" #include "XrdPosix/XrdPosixCache.hh" -#include "XrdPosix/XrdPosixCacheBC.hh" #include "XrdPosix/XrdPosixConfig.hh" #include "XrdPosix/XrdPosixFileRH.hh" #include "XrdPosix/XrdPosixInfo.hh" @@ -51,6 +49,8 @@ #include "XrdPosix/XrdPosixXrootd.hh" #include "XrdPosix/XrdPosixXrootdPath.hh" +#include "XrdRmc/XrdRmc.hh" + #include "XrdSys/XrdSysError.hh" #include "XrdSys/XrdSysTrace.hh" @@ -61,9 +61,7 @@ namespace XrdPosixGlobals { extern XrdScheduler *schedP; -extern XrdOucCache2 *theCache; -extern XrdOucCache *myCache; -extern XrdOucCache2 *myCache2; +extern XrdOucCache *theCache; extern XrdOucName2Name *theN2N; extern XrdCl::DirListFlags::Flags dlFlag; extern XrdSysLogger *theLogger; @@ -86,8 +84,8 @@ void XrdPosixConfig::EnvInfo(XrdOucEnv &theEnv) XrdPosixGlobals::schedP = (XrdScheduler *)theEnv.GetPtr("XrdScheduler*"); // If we have a new-style cache, propogate the environment to it -// - if (XrdPosixGlobals::myCache2) XrdPosixGlobals::myCache2->EnvInfo(theEnv); +//??? Get rid of EnvInfo +// if (XrdPosixGlobals::myCache2) XrdPosixGlobals::myCache2->EnvInfo(theEnv); } /******************************************************************************/ @@ -141,11 +139,10 @@ bool XrdPosixConfig::initCCM(XrdOucPsx &parms) void XrdPosixConfig::initEnv(char *eData) { - static XrdOucCacheDram dramCache; + static XrdRmc dramCache; + XrdRmc::Parms myParms; XrdOucEnv theEnv(eData); - XrdOucCache::Parms myParms; XrdOucCacheIO::aprParms apParms; - XrdOucCache *v1Cache; long long Val; char * tP; @@ -174,7 +171,7 @@ void XrdPosixConfig::initEnv(char *eData) // Get Mode // if ((tP = theEnv.Get("mode"))) - {if (*tP == 's') myParms.Options |= XrdOucCache::isServer; + {if (*tP == 's') myParms.Options |= XrdRmc::isServer; else if (*tP != 'c') DMSG("initEnv","'XRDPOSIX_CACHE=mode=" <Create(myParms, &apParms))) + myParms.Options |= XrdRmc::Serialized; + if (!(XrdPosixGlobals::theCache = dramCache.Create(myParms, &apParms))) {DMSG("initEnv", strerror(errno) <<" creating cache.");} - else XrdPosixGlobals::theCache = new XrdPosixCacheBC(v1Cache); } /******************************************************************************/ @@ -316,19 +308,14 @@ bool XrdPosixConfig::SetConfig(XrdOucPsx &parms) XrdPosixGlobals::ddInterval = (parms.cioWait < 10 ? 10 : parms.cioWait); } -// Handle the caching options +// Handle the caching options (library or builin memory). +// TODO: Make the memory cache a library plugin as well. // - if (parms.theCache2) - {XrdPosixGlobals::myCache2 = parms.theCache2; - XrdPosixGlobals::theCache = parms.theCache2; + if (parms.theCache) + {XrdPosixGlobals::theCache = parms.theCache; if (parms.initCCM) return initCCM(parms); return true; } - else if (parms.theCache) - {char ebuf[] = {0}; - XrdPosixGlobals::myCache = parms.theCache; - initEnv(ebuf); - } else if (parms.mCache && *parms.mCache) initEnv(parms.mCache); return true; diff --git a/src/XrdPosix/XrdPosixFile.cc b/src/XrdPosix/XrdPosixFile.cc index 6d3ec4e0d10..294d5e33b57 100644 --- a/src/XrdPosix/XrdPosixFile.cc +++ b/src/XrdPosix/XrdPosixFile.cc @@ -54,7 +54,7 @@ namespace XrdPosixGlobals { -extern XrdOucCache2 *theCache; +extern XrdOucCache *theCache; extern XrdOucName2Name *theN2N; extern XrdSysError *eDest; extern int ddInterval; @@ -90,7 +90,7 @@ int XrdPosixFile::ddNum = 0; XrdPosixFile::XrdPosixFile(bool &aOK, const char *path, XrdPosixCallBack *cbP, int Opts) - : XCio((XrdOucCacheIO2 *)this), PrepIO(0), + : XCio((XrdOucCacheIO *)this), PrepIO(0), mySize(0), myMtime(0), myInode(0), myMode(0), theCB(cbP), fLoc(0), cOpt(0), isStream(Opts & isStrm ? 1 : 0) @@ -123,8 +123,8 @@ XrdPosixFile::XrdPosixFile(bool &aOK, const char *path, XrdPosixCallBack *cbP, XrdPosixFile::~XrdPosixFile() { // Detach the cache if it is attached -// - if (XCio != this) XCio->Detach(); +//??? We need to figure this out, shouldn't need to do this +// if (XCio != this) XCio->Detach(); // Close the remote connection // @@ -148,17 +148,19 @@ XrdPosixFile::~XrdPosixFile() void* XrdPosixFile::DelayedDestroy(void* vpf) { // Static function. -// Called within a dedicated thread if XrdOucCacheIO is io-active or the -// file cannot be closed in a clean fashion for some reason. +// Called within a dedicated thread if there is a reference outstanding to the +// file or the file cannot be closed in a clean fashion for some reason. // EPNAME("DDestroy"); + XrdSysError *Say = XrdPosixGlobals::eDest; XrdCl::XRootDStatus Status; std::string statusMsg; const char *eTxt; XrdPosixFile *fCurr, *fNext; - int ddCount; - bool ioActive, doWait = false; + char buff[256]; + int ddCount, refNum; + bool doWait = false; // Wait for active I/O to complete // @@ -185,30 +187,32 @@ do{if (doWait) // Try to delete all the files on the list. If we exceeded the try limit, // remove the file from the list and let it sit forever. // + int nowLost = XrdPosixGlobals::ddNumLost; while((fCurr = fNext)) {fNext = fCurr->nextFile; - if (!((ioActive = fCurr->XCio->ioActive())) && !fCurr->Refs()) - {if (fCurr->Close(Status)) {delete fCurr; ddCount--; continue;} + if (!(refNum = fCurr->Refs())) + {if (fCurr->Close(Status) || !fCurr->clFile.IsOpen()) + {delete fCurr; ddCount--; continue;} else {statusMsg = Status.ToString(); eTxt = statusMsg.c_str(); } - } else eTxt = (ioActive ? "active I/O" : "callback"); + } else eTxt = 0; if (fCurr->numTries > XrdPosixGlobals::ddMaxTries) {XrdPosixGlobals::ddNumLost++; ddCount--; - if (XrdPosixGlobals::eDest) - {char buff[256]; - snprintf(buff, sizeof(buff), "(%d) %s", - XrdPosixGlobals:: ddNumLost, eTxt); - XrdPosixGlobals::eDest->Emsg("DDestroy", - "timeout closing", fCurr->Origin()); + if (!eTxt) + {snprintf(buff, sizeof(buff), "in use %d", refNum); + eTxt = buff; + } + if (Say) + {snprintf(buff, sizeof(buff), "%s timeout closing", eTxt); + Say->Emsg("DDestroy", buff, fCurr->Origin()); } else { DMSG("DDestroy", eTxt <<" timeout closing " <Origin() <<' ' <nextFile = ddLost; ddLost = fCurr; - fCurr->Close(Status); } else { fCurr->numTries++; doWait = true; @@ -218,7 +222,14 @@ do{if (doWait) ddMutex.UnLock(); } } - DEBUG("DLY destory end; "<= 3) + {snprintf(buff, sizeof(buff), "%d objects deferred and %d lost.", + ddCount, XrdPosixGlobals::ddNumLost); + Say->Emsg("DDestroy", buff); + } else { + DEBUG("DLY destory end; "<numTries = 0; + ddMutex.UnLock(); DEBUG("DLY destory "<<(doPost ? "post " : "has ")<Origin()); @@ -279,7 +290,7 @@ bool XrdPosixFile::Close(XrdCl::XRootDStatus &Status) bool XrdPosixFile::Finalize(XrdCl::XRootDStatus *Status) { - XrdOucCacheIO2 *ioP; + XrdOucCacheIO *ioP; // Indicate that we are at the start of the file // @@ -288,8 +299,8 @@ bool XrdPosixFile::Finalize(XrdCl::XRootDStatus *Status) // Complete initialization. If the stat() fails, the caller will unwind the // whole open process (ick). In the process get correct I/O vector. - if (!Status) ioP = (XrdOucCacheIO2 *)PrepIO; - else if (Stat(*Status)) ioP = (XrdOucCacheIO2 *)this; + if (!Status) ioP = (XrdOucCacheIO *)PrepIO; + else if (Stat(*Status)) ioP = (XrdOucCacheIO *)this; else return false; // Setup the cache if it is to be used @@ -364,7 +375,7 @@ const char *XrdPosixFile::Location() // If the file is not open, then we have no location // - if (!clFile.IsOpen()) return 0; + if (!clFile.IsOpen()) return ""; // If we have no location info, get it // diff --git a/src/XrdPosix/XrdPosixFile.hh b/src/XrdPosix/XrdPosixFile.hh index d8fa6739680..0687d010628 100644 --- a/src/XrdPosix/XrdPosixFile.hh +++ b/src/XrdPosix/XrdPosixFile.hh @@ -42,7 +42,7 @@ #include "XrdCl/XrdClURL.hh" #include "XrdCl/XrdClXRootDResponses.hh" -#include "XrdOuc/XrdOucCache2.hh" +#include "XrdOuc/XrdOucCache.hh" #include "XrdPosix/XrdPosixMap.hh" #include "XrdPosix/XrdPosixObject.hh" @@ -55,12 +55,13 @@ class XrdPosixCallBack; class XrdPosixPrepIO; class XrdPosixFile : public XrdPosixObject, - public XrdOucCacheIO2, + public XrdOucCacheIO, + public XrdOucCacheIOCD, public XrdCl::ResponseHandler { public: -XrdOucCacheIO2 *XCio; +XrdOucCacheIO *XCio; XrdPosixPrepIO *PrepIO; XrdCl::File clFile; @@ -82,6 +83,10 @@ static void DelayedDestroy(XrdPosixFile *fp); bool Close(XrdCl::XRootDStatus &Status); + bool Detach(XrdOucCacheIOCD &cdP) {(void)cdP; return true;} + + void DetachDone() {unRef();} + bool Finalize(XrdCl::XRootDStatus *Status); long long FSize() {AtomicBeg(updMutex); @@ -97,8 +102,6 @@ static void DelayedDestroy(XrdPosixFile *fp); void HandleResponse(XrdCl::XRootDStatus *status, XrdCl::AnyObject *response); - void isOpen(); - void updLock() {updMutex.Lock();} void updUnLock() {updMutex.UnLock();} diff --git a/src/XrdPosix/XrdPosixPrepIO.hh b/src/XrdPosix/XrdPosixPrepIO.hh index e3fda717f73..e836141d6b2 100644 --- a/src/XrdPosix/XrdPosixPrepIO.hh +++ b/src/XrdPosix/XrdPosixPrepIO.hh @@ -33,13 +33,13 @@ #include "Xrd/XrdJob.hh" #include "XrdPosix/XrdPosixFile.hh" -class XrdPosixPrepIO : public XrdOucCacheIO2 +class XrdOucCacheIOCD; + +class XrdPosixPrepIO : public XrdOucCacheIO { public: -XrdOucCacheIO *Base() {return this;} // Already defined - -XrdOucCacheIO *Detach() {return this;} // Already defined +bool Detach(XrdOucCacheIOCD &cdP) {(void)cdP; return true;} void Disable(); diff --git a/src/XrdPosix/XrdPosixXrootd.cc b/src/XrdPosix/XrdPosixXrootd.cc index 39f5597d00b..0d2387cb373 100644 --- a/src/XrdPosix/XrdPosixXrootd.cc +++ b/src/XrdPosix/XrdPosixXrootd.cc @@ -50,14 +50,12 @@ #include "XrdSys/XrdSysHeaders.hh" #include "XrdSys/XrdSysPlatform.hh" -#include "XrdOuc/XrdOucCache2.hh" -#include "XrdOuc/XrdOucCacheDram.hh" +#include "XrdOuc/XrdOucCache.hh" #include "XrdOuc/XrdOucEnv.hh" #include "XrdOuc/XrdOucName2Name.hh" #include "XrdOuc/XrdOucPsx.hh" #include "XrdPosix/XrdPosixAdmin.hh" -#include "XrdPosix/XrdPosixCacheBC.hh" #include "XrdPosix/XrdPosixCallBack.hh" #include "XrdPosix/XrdPosixConfig.hh" #include "XrdPosix/XrdPosixDir.hh" @@ -81,9 +79,7 @@ class XrdSysError; namespace XrdPosixGlobals { XrdScheduler *schedP = 0; -XrdOucCache2 *theCache = 0; -XrdOucCache *myCache = 0; -XrdOucCache2 *myCache2 = 0; +XrdOucCache *theCache = 0; XrdOucName2Name *theN2N = 0; XrdCl::DirListFlags::Flags dlFlag = XrdCl::DirListFlags::None; XrdSysLogger *theLogger = 0; @@ -265,16 +261,21 @@ int XrdPosixXrootd::Close(int fildes) if (!(fP = XrdPosixObject::ReleaseFile(fildes))) {errno = EBADF; return -1;} -// Close the file if there is no active I/O (possible caching). Delete the -// object if the close was successful (it might not be). +// Detach the file from a possible cache. We need to up the reference count +// to synchrnoize with any possible callback as we may need to place this +// object in he delayed destroy queue if it is stil being used. Note that +// the caller will get a zero return code should we delay the close. // - if (!(fP->XCio->ioActive()) && !fP->Refs()) + fP->Ref(); + if (fP->XCio->Detach((XrdOucCacheIOCD&)*fP) && fP->Refs() < 2) {if ((ret = fP->Close(Status))) {delete fP; fP = 0;} else if (DEBUGON) {std::string eTxt = Status.ToString(); DEBUG(eTxt <<" closing " <Origin()); } - } else ret = true; + } else { + ret = true; + } // If we still have a handle then we need to do a delayed delete on this // object because either the close failed or there is still active I/O @@ -593,14 +594,14 @@ int XrdPosixXrootd::Open(const char *path, int oflags, mode_t mode, // If we have a cache, then issue a prepare as the cache may want to defer the // open request ans we have a lot more work to do. // - if (XrdPosixGlobals::myCache2) + if (XrdPosixGlobals::theCache) {int rc; if (infoP && isRO && OpenCache(*fp, *infoP)) {delete fp; errno = 0; return -3; } - rc = XrdPosixGlobals::myCache2->Prepare(fp->Path(), oflags, mode); + rc = XrdPosixGlobals::theCache->Prepare(fp->Path(), oflags, mode); if (rc > 0) return OpenDefer(fp, cbP, XOflags, XOmode, oflags&isStream); if (rc < 0) {delete fp; errno = -rc; return -1;} } @@ -652,10 +653,9 @@ bool XrdPosixXrootd::OpenCache(XrdPosixFile &file,XrdPosixInfo &Info) // Check if the full file is in the cache // - rc = XrdPosixGlobals::myCache2->LocalFilePath(file.Path(), Info.cachePath, - sizeof(Info.cachePath), - XrdOucCache2::ForAccess, - Info.ffReady); + rc = XrdPosixGlobals::theCache->LocalFilePath(file.Path(), Info.cachePath, + (int)sizeof(Info.cachePath), + XrdOucCache::ForAccess); if (rc == 0) {Info.ffReady = true; DEBUG("File in cache url=" <Stat(statX.path, *buf); + int rc = XrdPosixGlobals::theCache->Stat(statX.path, *buf); if (!rc) return 0; if (rc < 0) {errno = -rc; return -1;} } diff --git a/src/XrdPosix/XrdPosixXrootd.hh b/src/XrdPosix/XrdPosixXrootd.hh index 970e9331504..1be5949046b 100644 --- a/src/XrdPosix/XrdPosixXrootd.hh +++ b/src/XrdPosix/XrdPosixXrootd.hh @@ -49,12 +49,6 @@ struct XrdOucIOVec; -class XrdScheduler; -class XrdOucCache; -class XrdOucCache2; -class XrdOucEnv; -class XrdOucName2Name; -class XrdSysLogger; class XrdPosixCallBack; class XrdPosixCallBackIO; class XrdPosixFile; diff --git a/src/XrdPss/XrdPssConfig.cc b/src/XrdPss/XrdPssConfig.cc index 024bfc0b2b0..49e4252ebad 100644 --- a/src/XrdPss/XrdPssConfig.cc +++ b/src/XrdPss/XrdPssConfig.cc @@ -53,7 +53,7 @@ #include "XrdSys/XrdSysPthread.hh" #include "XrdOuc/XrdOuca2x.hh" -#include "XrdOuc/XrdOucCache2.hh" +#include "XrdOuc/XrdOucCache.hh" #include "XrdOuc/XrdOucEnv.hh" #include "XrdOuc/XrdOucExport.hh" #include "XrdOuc/XrdOucN2NLoader.hh" @@ -61,7 +61,6 @@ #include "XrdOuc/XrdOucStream.hh" #include "XrdOuc/XrdOucTList.hh" #include "XrdOuc/XrdOucUtils.hh" -#include "XrdOuc/XrdOucCache2.hh" #include "XrdPosix/XrdPosixConfig.hh" #include "XrdPosix/XrdPosixXrootd.hh" @@ -217,14 +216,14 @@ int XrdPssSys::Configure(const char *cfn) // if (psxConfig->xLfn2Pfn) xLfn2Pfn = (theN2N = psxConfig->theN2N) != 0; -// If we have a version 2 cache then save it and check if we need to tell +// If we have a cache then save it and check if we need to tell // xrootd we allow a redirect on a read (this is complicated). -// - if (psxConfig->theCache2 && dcaCTime) - {char buff[32]; - sprintf(buff, "%d", dcaCTime); - XrdOucEnv::Export("XRDXROOTD_CACHERDRDR", buff); - } +// ??? Why are we doing this +// if (psxConfig->theCache2 && dcaCTime) +// {char buff[32]; +// sprintf(buff, "%d", dcaCTime); +// XrdOucEnv::Export("XRDXROOTD_CACHERDRDR", buff); +// } // All done with the configurator // diff --git a/src/XrdRmc/XrdRmc.cc b/src/XrdRmc/XrdRmc.cc new file mode 100644 index 00000000000..32dfb078b4c --- /dev/null +++ b/src/XrdRmc/XrdRmc.cc @@ -0,0 +1,52 @@ +/******************************************************************************/ +/* */ +/* X r d R m c . c c */ +/* */ +/* (c) 2019 by the Board of Trustees of the Leland Stanford, Jr., University */ +/* All Rights Reserved */ +/* Produced by Andrew Hanushevsky for Stanford University under contract */ +/* DE-AC02-76-SFO0515 with the Department of Energy */ +/* */ +/* This file is part of the XRootD software suite. */ +/* */ +/* XRootD is free software: you can redistribute it and/or modify it under */ +/* the terms of the GNU Lesser General Public License as published by the */ +/* Free Software Foundation, either version 3 of the License, or (at your */ +/* option) any later version. */ +/* */ +/* XRootD is distributed in the hope that it will be useful, but WITHOUT */ +/* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ +/* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ +/* License for more details. */ +/* */ +/* You should have received a copy of the GNU Lesser General Public License */ +/* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ +/* COPYING (GPL license). If not, see . */ +/* */ +/* The copyright holder's institutional names and contributor's names may not */ +/* be used to endorse or promote products derived from this software without */ +/* specific prior written permission of the institution or contributor. */ +/******************************************************************************/ + +#include + +#include "XrdRmc/XrdRmc.hh" +#include "XrdRmc/XrdRmcReal.hh" + +/******************************************************************************/ +/* C r e a t e */ +/******************************************************************************/ + +XrdOucCache *XrdRmc::Create(Parms &ParmV, XrdOucCacheIO::aprParms *aprP) +{ + XrdRmcReal *cP; + int rc; + +// We simply create a new instance of a real cache and return it. We do it this +// way so that in the future new types of caches can be created using the same +// interface. +// + cP = new XrdRmcReal(rc, ParmV, aprP); + if (rc) {delete cP; cP = 0; errno = rc;} + return (XrdOucCache *)cP; +} diff --git a/src/XrdRmc/XrdRmc.hh b/src/XrdRmc/XrdRmc.hh new file mode 100644 index 00000000000..02c538a2888 --- /dev/null +++ b/src/XrdRmc/XrdRmc.hh @@ -0,0 +1,151 @@ +#ifndef __XRDRMC_HH__ +#define __XRDRMC_HH__ +/******************************************************************************/ +/* */ +/* X r d R m c . h h */ +/* */ +/* (c) 2019 by the Board of Trustees of the Leland Stanford, Jr., University */ +/* All Rights Reserved */ +/* Produced by Andrew Hanushevsky for Stanford University under contract */ +/* DE-AC02-76-SFO0515 with the Department of Energy */ +/* */ +/* This file is part of the XRootD software suite. */ +/* */ +/* XRootD is free software: you can redistribute it and/or modify it under */ +/* the terms of the GNU Lesser General Public License as published by the */ +/* Free Software Foundation, either version 3 of the License, or (at your */ +/* option) any later version. */ +/* */ +/* XRootD is distributed in the hope that it will be useful, but WITHOUT */ +/* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ +/* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ +/* License for more details. */ +/* */ +/* You should have received a copy of the GNU Lesser General Public License */ +/* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ +/* COPYING (GPL license). If not, see . */ +/* */ +/* The copyright holder's institutional names and contributor's names may not */ +/* be used to endorse or promote products derived from this software without */ +/* specific prior written permission of the institution or contributor. */ +/******************************************************************************/ + +#include "XrdOuc/XrdOucCache.hh" + +/*! The class defined here implements a general memory cache for data from an + arbitrary source (e.g. files, sockets, etc). It is based on the abstract + definition of a cache. Use the Create() method to create instances of a + cache. There can be many such instances. Each instance is associated with + one or more XrdOucCacheIO objects (see the XrdOucCache::Attach() method). + + Notes: 1. The minimum PageSize is 4096 (4k) and must be a power of 2. + The maximum PageSize is 16MB. + 2. The size of the cache is forced to be a multiple PageSize and + have a minimum size of PageSize * 256. + 3. The minimum external read size is equal to PageSize. + 4. Currently, only write-through caches are supported. + 5. The Max2Cache value avoids placing data in the cache when a read + exceeds the specified value. The minimum allowed is PageSize, which + is also the default. + 6. Structured file optimization allows pages whose bytes have been + fully referenced to be discarded; effectively increasing the cache. + 7. A structured cache treats all files as structured. By default, the + cache treats files as unstructured. You can over-ride the settings + on an individual file basis when the file's I/O object is attached + by passing the XrdOucCache::optFIS option, if needed. + 8. Write-in caches are only supported for files attached with the + XrdOucCache::optWIN setting. Otherwise, updates are handled + with write-through operations. + 9. A cache object may be deleted. However, the deletion is delayed + until all CacheIO objects attached to the cache are detached. + 10. The default maximum attached files is set to 8192 when isServer + has been specified. Otherwise, it is set at 256. + 11. When canPreRead is specified, the cache asynchronously handles + preread requests (see XrdOucCacheIO::Preread()) using 9 threads + when isServer is in effect. Otherwise, 3 threads are used. + 12. The max queue depth for prereads is 8. When the max is exceeded + the oldest preread is discarded to make room for the newest one. + 13. If you specify the canPreRead option when creating the cache you + can also enable automatic prereads if the algorithm is workable. + Otherwise, you will need to implement your own algorithm and + issue prereads manually using the XrdOucCacheIO::Preread() method. + 14. The automatic preread algorithm is (see aprParms): + a) A preread operation occurs when all of the following conditions + are satisfied: + o The cache CanPreRead option is in effect. + o The read length < 'miniRead' + ||(read length < 'maxiRead' && Offset == next maxi offset) + b) The preread page count is set to be readlen/pagesize and the + preread occurs at the page after read_offset+readlen. The page + is adjusted, as follows: + o If the count is < minPages, it is set to minPages. + o The count must be > 0 at this point. + c) Normally, pre-read pages participate in the LRU scheme. However, + if the preread was triggered using 'maxiRead' then the pages are + marked for single use only. This means that the moment data is + delivered from the page, the page is recycled. + 15. Invalid options silently force the use of the default. +*/ + +class XrdRmc +{ +public: + +//----------------------------------------------------------------------------- +//! Parameters for a newly created memory cache. +//----------------------------------------------------------------------------- + +struct Parms + {long long CacheSize; //!< Size of cache in bytes (default 100MB) + int PageSize; //!< Size of each page in bytes (default 32KB) + int Max2Cache; //!< Largest read to cache (default PageSize) + int MaxFiles; //!< Maximum number of files (default 256 or 8K) + int Options; //!< Options as defined below (default r/o cache) + short minPages; //!< Minimum number of pages (default 256) + short Reserve1; //!< Reserved for future use + int Reserve2; //!< Reserved for future use + + Parms() : CacheSize(104857600), PageSize(32768), + Max2Cache(0), MaxFiles(0), Options(0), + minPages(0), Reserve1(0), Reserve2(0) {} + }; + +// Valid option values in Parms::Options +// +static const int +isServer = 0x0010; //!< This is server application; not a user application. + // Appropriate internal optimizations will be used. +static const int +isStructured = 0x0020; // Optimize for structured files (e.g. root). + +static const int +canPreRead = 0x0040; //!< Enable pre-read operations (o/w ignored) + +static const int +logStats = 0x0080; //!< Display statistics upon detach + +static const int +Serialized = 0x0004; //!< Caller ensures MRSW semantics + +static const int +ioMTSafe = 0x0008; //!< CacheIO object is MT-safe + +static const int +Debug = 0x0003; //!< Produce some debug messages (levels 0, 1, 2, or 3) + +//----------------------------------------------------------------------------- +//! Create an instance of a memory cache. +//! +//! @param Reference to mandatory cache parameters. +//! @param Optional pointer to default automatic preread parameters. +//! +//! @return Success: a pointer to a new instance of the cache. +//! Failure: a null pointer is returned and errno set to the reason. +//----------------------------------------------------------------------------- + +static XrdOucCache *Create(Parms &Params, XrdOucCacheIO::aprParms *aprP=0); + + XrdRmc() {} + ~XrdRmc() {} +}; +#endif diff --git a/src/XrdRmc/XrdRmcData.cc b/src/XrdRmc/XrdRmcData.cc new file mode 100644 index 00000000000..ffa49633d22 --- /dev/null +++ b/src/XrdRmc/XrdRmcData.cc @@ -0,0 +1,581 @@ +/******************************************************************************/ +/* */ +/* X r d R m c D a t a . c c */ +/* */ +/* (c) 2019 by the Board of Trustees of the Leland Stanford, Jr., University */ +/* All Rights Reserved */ +/* Produced by Andrew Hanushevsky for Stanford University under contract */ +/* DE-AC02-76-SFO0515 with the Department of Energy */ +/* */ +/* This file is part of the XRootD software suite. */ +/* */ +/* XRootD is free software: you can redistribute it and/or modify it under */ +/* the terms of the GNU Lesser General Public License as published by the */ +/* Free Software Foundation, either version 3 of the License, or (at your */ +/* option) any later version. */ +/* */ +/* XRootD is distributed in the hope that it will be useful, but WITHOUT */ +/* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ +/* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ +/* License for more details. */ +/* */ +/* You should have received a copy of the GNU Lesser General Public License */ +/* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ +/* COPYING (GPL license). If not, see . */ +/* */ +/* The copyright holder's institutional names and contributor's names may not */ +/* be used to endorse or promote products derived from this software without */ +/* specific prior written permission of the institution or contributor. */ +/******************************************************************************/ + +#include +#include + +#include "XrdRmc/XrdRmc.hh" +#include "XrdRmc/XrdRmcData.hh" +#include "XrdSys/XrdSysHeaders.hh" + +/******************************************************************************/ +/* C o n s t r u c t o r */ +/******************************************************************************/ + +XrdRmcData::XrdRmcData(XrdRmcReal *cP, XrdOucCacheIO *ioP, + long long vn, int opts) + : pPLock(0), rPLock(0), wPLock(0), + Cache(cP), ioObj(ioP), VNum(vn) +{ +// We need to map the cache options to our local options +// + isFIS = (opts & XrdOucCache::optFIS ? 1 : 0); + isRW = (opts & XrdOucCache::optRW ? okRW : 0); + +// Copy some values from the cache to our local area for convenience +// + SegShft = Cache->SegShft; + OffMask = Cache->OffMask; + SegSize = Cache->SegSize; + maxCache = Cache->maxCache; + Debug = Cache->Dbg; + +// Initialize the pre-read area +// + memset(prRR, -1, sizeof(prRR) ); + memset(prBeg, -1, sizeof(prBeg)); + memset(prEnd, -1, sizeof(prEnd)); + memset(prOpt, 0, sizeof(prOpt)); + + prNSS =-1; + prRRNow = 0; + prStop = 0; + prNext = prFree = 0; + prActive = 0; + prOK = (Cache->prNum ? 1 : 0); + prReq.Data = this; + prAuto = (prOK ? setAPR(Apr, Cache->aprDefault, SegSize) : 0); + prPerf = 0; + prCalc = Apr.prRecalc; + +// Establish serialization options +// + if (Cache->Options & XrdRmc::ioMTSafe) pPLopt = rPLopt = xs_Shared; + else pPLopt = rPLopt = xs_Exclusive; + +// Establish serialization handling (only needed for r/w files) +// + if (Cache->Options & XrdRmc::Serialized) + {if (Cache->Options & XrdRmc::ioMTSafe) + {if (isRW && prOK) pPLock = wPLock = &rwLock;} + else if (prOK) rPLock = pPLock = wPLock = &rwLock; + } else if (!(Cache->Options & XrdRmc::ioMTSafe) || isRW) + rPLock = pPLock = wPLock = &rwLock; +} + +/******************************************************************************/ +/* D e t a c h */ +/******************************************************************************/ + +bool XrdRmcData::Detach(XrdOucCacheIOCD &iocd) +{ + int delOK; + +// We must wait for any pre-reads to stop at this point. TO DO: We really +// should run this in a sperate thread and use the callback mechanism. +// + DMutex.Lock(); + if (prActive) + {XrdSysSemaphore prDone(0); + prStop = &prDone; + DMutex.UnLock(); + prDone.Wait(); + DMutex.Lock(); + } + +// Get exclusive control +// + rwLock.Lock(xs_Exclusive); + +// We can now detach ourselves from the cache +// + delOK = Cache->Detach(ioObj); + DMutex.UnLock(); + rwLock.UnLock(xs_Exclusive); + +// Check if we should delete ourselves and if so add our stats to the cache +// + if (delOK) + {Cache->Stats.Add(Statistics); + if (Cache->Lgs) + {char sBuff[4096]; + snprintf(sBuff, sizeof(sBuff), + "Cache: Stats: %lld Read; %lld Get; %lld Pass; " + "%lld Write; %lld Put; %lld Hits; %lld Miss; " + "%lld pead; %lld HitsPR; %lld MissPR; Path %s\n", + Statistics.BytesRead, Statistics.BytesGet, + Statistics.BytesPass, Statistics.BytesWrite, + Statistics.BytesPut, + Statistics.Hits, Statistics.Miss, + Statistics.BytesPead, + Statistics.HitsPR, Statistics.MissPR, + ioObj->Path()); + cerr <Post(); + DMutex.UnLock(); + return; + } + +// Do the next pre-read in the queue (it's possible another may get in) +// +do{if ((oVal = prOpt[prNext])) + {segBeg = prBeg[prNext]; segEnd = prEnd[prNext]; + prOpt[prNext++] = 0; + if (prNext >= prMax) prNext = 0; + if (oVal == prSKIP) continue; + prActive = prRun; + if (Debug > 1) cerr <<"prD: beg " <<(VNum >>XrdRmcReal::Shift) <<' ' + <<(segEnd-segBeg+1)*SegSize <<'@' <<(segBeg*SegSize) + <<" f=" <Path() <Get(ioObj, segBeg, rLen, noIO))) + {if (noIO) pVal = 0; + else {pVal = oVal; bPead += rLen; prPages++;} + } + } while(cBuff && Cache->Ref(cBuff, 0, pVal) && segBeg++ < segEnd); + if (Debug > 1) cerr <<"PrD: end " <<(VNum >>XrdRmcReal::Shift) + <<' ' <Post(); + } else if (prOpt[prNext]) + {prActive = prWait; + Cache->PreRead(&prReq); + } else prActive = 0; + +// All done here +// + DMutex.UnLock(); +} + +/******************************************************************************/ + +void XrdRmcData::Preread(long long Offs, int rLen, int Opts) +{ + int How; + +// Determine how to place the pages. We do this via assignment to avoid a gcc +// bug that doesn't optimize out static const int's except via assignment. +// + if (Opts & SingleUse) How = prSUSE; + else How = prLRU; + +// Verify that this preread will succeed then schedule it if so +// + if (prOK && rLen > 0 && Offs > 0 + && Offs < XrdRmcReal::MaxFO && (Offs + rLen) < XrdRmcReal::MaxFO) return; + QueuePR(Offs>>SegShft, rLen, How); +} + +/******************************************************************************/ + +void XrdRmcData::Preread(aprParms &Parms) +{ + +// Establish the new feature set if prereads are enabled +// + if (prOK) + {DMutex.Lock(); + prAuto = setAPR(Apr, Parms, SegSize); + DMutex.UnLock(); + } +} + +/******************************************************************************/ +/* Q u e u e P R */ +/******************************************************************************/ + +void XrdRmcData::QueuePR(long long segBeg, int rLen, int prHow, int isAuto) +{ + XrdSysMutexHelper Monitor(&DMutex); + long long segCnt, segEnd; + int i; + +// Scuttle everything if we are stopping +// + if (Debug) cerr <<"prQ: req " < prBeg[i] && segEnd <= prEnd[i])) + {if (prHow == prSKIP) + {if (Debug) cerr <<"pDQ: " < prCalc) + {int crPerf; + Statistics.Lock(); + prCalc = Statistics.BytesPead + Apr.prRecalc; + crPerf = (Statistics.MissPR?(Statistics.HitsPR*100)/Statistics.MissPR:0); + Statistics.UnLock(); + if (Debug) cerr <<"PrD: perf " <Path() <= 0) + {if ( crPerf < Apr.minPerf && prPerf < Apr.minPerf + && (crPerf <= prPerf || crPerf <= prPerf*2)) + {if (Debug) cerr <<"PrD: Disabled for " <Path() <= prMax) prFree = 0; + +// If nothing pending then activate a preread +// + if (Debug) cerr <<"prQ: add " <PreRead(&prReq);} +} + +/******************************************************************************/ +/* R e a d */ +/******************************************************************************/ + +int XrdRmcData::Read(char *Buff, long long Offs, int rLen) +{ + MrSw EnforceMrSw(rPLock, rPLopt); + XrdOucCacheStats Now; + char *cBuff, *Dest = Buff; + long long segOff, segNum = (Offs >> SegShft); + int noIO, rAmt, rGot, doPR = prAuto, rLeft = rLen; + +// Verify read length and offset +// + if (rLen <= 0) return 0; + if (XrdRmcReal::MaxFO < Offs || Offs < 0 + || XrdRmcReal::MaxFO < (Offs + rLen)) return -EOVERFLOW; + +// Check for preread request and Determine how to place the pages. +// + if (!Buff) + {int How; + if (rLen > maxCache) How = prSUSE; + else How = prLRU; + QueuePR(segNum, rLen, How); + return 0; + } + +// Ignore caching it if it's too large. Use alternate read algorithm. +// + if (rLen > maxCache) return Read(Now, Buff, Offs, rLen); + +// We check now whether or not we will try to do a preread later. This is +// advisory at this point so we don't need to obtain any locks to do this. +// + if (doPR) + {if (rLen >= Apr.Trigger) doPR = 0; + else for (noIO = 0; noIO < prRRMax; noIO++) + if (prRR[noIO] == segNum) {doPR = 0; break;} + if (doPR) + {DMutex.Lock(); + prRR[prRRNow] = segNum; + prRRNow = (prRRNow+1)%prRRMax; + DMutex.UnLock(); + } + } + if (Debug > 1) cerr <<"Rdr: " <> SegShft) | VNum; + segOff = Offs & OffMask; + rAmt = SegSize - segOff; + if (rAmt > rLen) rAmt = rLen; + if (Debug > 1) cerr <<"Rdr: " <100) Dest.minPerf = 100; + + +// Indicate whether anything can be preread +// + return (Dest.minPages > 0 && Dest.Trigger > 1); +} + +/******************************************************************************/ +/* T r u n c */ +/******************************************************************************/ + +int XrdRmcData::Trunc(long long Offs) +{ + MrSw EnforceMrSw(wPLock, xs_Exclusive); + +// Verify that we can modify this cache and the trunc offset +// + if (!isRW) return -EROFS; + if (Offs > XrdRmcReal::MaxFO || Offs < 0) return -EOVERFLOW; + +// Get the segment pointer and truncate pages from the cache +// + Cache->Trunc(ioObj, (Offs >> SegShft) | VNum); + +// Now just return the result of actually doing the truncate +// + return ioObj->Trunc(Offs); +} + +/******************************************************************************/ +/* W r i t e */ +/******************************************************************************/ + +int XrdRmcData::Write(char *Buff, long long Offs, int wLen) +{ + MrSw EnforceMrSw(wPLock, xs_Exclusive); + XrdOucCacheStats Now; + char *cBuff, *Src = Buff; + long long segOff, segNum; + int noIO, wAmt, rGot, wLeft = wLen; + +// Verify write length, ability, and buffer +// + if (wLen <= 0) return 0; + if (!isRW) return -EROFS; + if (!Buff) return -EINVAL; + if (XrdRmcReal::MaxFO < Offs || Offs < 0 + || XrdRmcReal::MaxFO < (Offs + wLen)) return -EOVERFLOW; + +// First step is to write out all the data (write-through for now) +// + if ((wAmt = ioObj->Write(Buff, Offs, wLen)) != wLen) + return (wAmt < 0 ? wAmt : -EIO); + Now.BytesWrite = wLen; + +// Get the segment pointer, offset and the initial write amount +// + segNum = (Offs >> SegShft) | VNum; + segOff = Offs & OffMask; + wAmt = SegSize - segOff; + if (wAmt > wLen) wAmt = wLen; + +// Now update any pages that are actually in the cache +// +do{if (!(cBuff = Cache->Get(0, segNum, rGot, noIO))) Now.Miss++; + else {memcpy(cBuff+segOff, Src, wAmt); + Now.BytesPut += wAmt; Now.Hits++; + if (noIO < 0) Now.HitsPR++; + Cache->Upd(cBuff, wAmt, segOff); + } + Src += wAmt; + if ((wLeft -= wAmt) <= 0) break; + wAmt = (wLeft <= SegSize ? wLeft : SegSize); + segNum++; segOff = 0; + } while(1); + +// Update stats and return how much we wrote +// + Statistics.Add(Now); + return wLen; +} diff --git a/src/XrdRmc/XrdRmcData.hh b/src/XrdRmc/XrdRmcData.hh new file mode 100644 index 00000000000..3e2db441d71 --- /dev/null +++ b/src/XrdRmc/XrdRmcData.hh @@ -0,0 +1,148 @@ +#ifndef __XRDRMCDATA_HH__ +#define __XRDRMCDATA_HH__ +/******************************************************************************/ +/* */ +/* X r d R m c D a t a . h h */ +/* */ +/* (c) 2019 by the Board of Trustees of the Leland Stanford, Jr., University */ +/* All Rights Reserved */ +/* Produced by Andrew Hanushevsky for Stanford University under contract */ +/* DE-AC02-76-SFO0515 with the Department of Energy */ +/* */ +/* This file is part of the XRootD software suite. */ +/* */ +/* XRootD is free software: you can redistribute it and/or modify it under */ +/* the terms of the GNU Lesser General Public License as published by the */ +/* Free Software Foundation, either version 3 of the License, or (at your */ +/* option) any later version. */ +/* */ +/* XRootD is distributed in the hope that it will be useful, but WITHOUT */ +/* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ +/* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ +/* License for more details. */ +/* */ +/* You should have received a copy of the GNU Lesser General Public License */ +/* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ +/* COPYING (GPL license). If not, see . */ +/* */ +/* The copyright holder's institutional names and contributor's names may not */ +/* be used to endorse or promote products derived from this software without */ +/* specific prior written permission of the institution or contributor. */ +/******************************************************************************/ + +/* The XrdRmcData object defines a remanufactured XrdOucCacheIO object and + is used to front a XrdOucCacheIO object with an XrdRmcReal object. +*/ + +#include "XrdOuc/XrdOucCache.hh" +#include "XrdRmc/XrdRmcReal.hh" +#include "XrdSys/XrdSysPthread.hh" +#include "XrdSys/XrdSysXSLock.hh" + +class XrdRmcData : public XrdOucCacheIO +{ +public: + +bool Detach(XrdOucCacheIOCD &iocd); + +long long FSize() {return (ioObj ? ioObj->FSize() : 0);} + +const char *Path() {return ioObj->Path();} + +void Preread(); + +void Preread(aprParms &Parms); + +void Preread(long long Offs, int rLen, int Opts=0); + +int Read (char *Buffer, long long Offset, int Length); + +static int setAPR(aprParms &Dest, aprParms &Src, int pSize); + +int Sync() {return 0;} // We only support write-through for now + +int Trunc(long long Offset); + +int Write(char *Buffer, long long Offset, int Length); + + XrdRmcData(XrdRmcReal *cP, XrdOucCacheIO *ioP, + long long vn, int opts); + +private: + ~XrdRmcData() {} +void QueuePR(long long SegOffs, int rLen, int prHow, int isAuto=0); +int Read (XrdOucCacheStats &Now, + char *Buffer, long long Offs, int Length); + +// The following is for read/write support +// +class MrSw +{ +public: +inline void UnLock() {if (myLock) {myLock->UnLock(myUsage); myLock = 0;}} + + MrSw(XrdSysXSLock *lP, XrdSysXS_Type usage) : myUsage(usage) + {if ((myLock = lP)) lP->Lock(usage);} + ~MrSw() {if (myLock) myLock->UnLock(myUsage);} + +private: +XrdSysXSLock *myLock; +XrdSysXS_Type myUsage; +}; + +// The following supports MRSW serialization +// +XrdSysXSLock rwLock; +XrdSysXSLock *pPLock; // 0 if no preread lock required +XrdSysXSLock *rPLock; // 0 if no read lock required +XrdSysXSLock *wPLock; // 0 if no write lock required +XrdSysXS_Type pPLopt; +XrdSysXS_Type rPLopt; + +XrdSysMutex DMutex; +XrdRmcReal *Cache; +XrdOucCacheIO *ioObj; +long long VNum; +long long SegSize; +long long OffMask; +long long SegShft; +int maxCache; +char isFIS; +char isRW; +char Debug; + +static const int okRW = 1; +static const int xqRW = 2; + +// Preread Control Area +// +XrdRmcReal::prTask prReq; +XrdSysSemaphore *prStop; + +long long prNSS; // Next Sequential Segment for maxi prereads + +static const int prRRMax= 5; +long long prRR[prRRMax]; // Recent reads +int prRRNow; // Pointer to next entry to use + +static const int prMax = 8; +static const int prRun = 1; // Status in prActive (running) +static const int prWait = 2; // Status in prActive (waiting) + +static const int prLRU = 1; // Status in prOpt (set LRU) +static const int prSUSE = 2; // Status in prOpt (set Single Use) +static const int prSKIP = 3; // Status in prOpt (skip entry) + +aprParms Apr; +long long prCalc; +long long prBeg[prMax]; +long long prEnd[prMax]; +int prNext; +int prFree; +int prPerf; +char prOpt[prMax]; +char prOK; +char prActive; +char prAuto; +}; +#endif diff --git a/src/XrdRmc/XrdRmcReal.cc b/src/XrdRmc/XrdRmcReal.cc new file mode 100644 index 00000000000..a10d4629770 --- /dev/null +++ b/src/XrdRmc/XrdRmcReal.cc @@ -0,0 +1,599 @@ +/******************************************************************************/ +/* */ +/* X r d R m c R e a l . c c */ +/* */ +/* (c) 2019 by the Board of Trustees of the Leland Stanford, Jr., University */ +/* All Rights Reserved */ +/* Produced by Andrew Hanushevsky for Stanford University under contract */ +/* DE-AC02-76-SFO0515 with the Department of Energy */ +/* */ +/* This file is part of the XRootD software suite. */ +/* */ +/* XRootD is free software: you can redistribute it and/or modify it under */ +/* the terms of the GNU Lesser General Public License as published by the */ +/* Free Software Foundation, either version 3 of the License, or (at your */ +/* option) any later version. */ +/* */ +/* XRootD is distributed in the hope that it will be useful, but WITHOUT */ +/* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ +/* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ +/* License for more details. */ +/* */ +/* You should have received a copy of the GNU Lesser General Public License */ +/* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ +/* COPYING (GPL license). If not, see . */ +/* */ +/* The copyright holder's institutional names and contributor's names may not */ +/* be used to endorse or promote products derived from this software without */ +/* specific prior written permission of the institution or contributor. */ +/******************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include + +#include "XrdRmc/XrdRmcData.hh" +#include "XrdSys/XrdSysHeaders.hh" + +#ifdef __APPLE__ +#ifndef MAP_ANONYMOUS +#define MAP_ANONYMOUS MAP_ANON +#endif +#endif + +/******************************************************************************/ +/* C o n s t r u c t o r */ +/******************************************************************************/ + +void *XrdRmcRealPRXeq(void *parg) +{ XrdRmcReal *cP = (XrdRmcReal *)parg; + cP->PreRead(); + return (void *)0; +} + +XrdRmcReal::XrdRmcReal(int &rc, XrdRmc::Parms &ParmV, + XrdOucCacheIO::aprParms *aprP) + : Slots(0), Slash(0), Base((char *)MAP_FAILED), Dbg(0), Lgs(0), + AZero(0), Attached(0), prFirst(0), prLast(0), + prReady(0), prStop(0), prNum(0) +{ + size_t Bytes; + int n, minPag, isServ = ParmV.Options & XrdRmc::isServer; + +// Copy over options +// + rc = ENOMEM; + Options = ParmV.Options; + if (Options & XrdRmc::Debug) Lgs = Dbg = (Options & XrdRmc::Debug); + if (Options & XrdRmc::logStats) Lgs = 1; + minPag = (ParmV.minPages <= 0 ? 256 : ParmV.minPages); + +// Establish maximum number of attached files +// + if (ParmV.MaxFiles <= 0) maxFiles = (isServ ? 16384 : 256); + else {maxFiles = (ParmV.MaxFiles > 32764 ? 32764 : ParmV.MaxFiles); + maxFiles = maxFiles/sizeof(int)*sizeof(int); + if (!maxFiles) maxFiles = 256; + } + +// Adjust segment size to be a power of two and atleast 4k. +// + if (ParmV.PageSize <= 0) SegSize = 32768; + else if (!(SegSize = ParmV.PageSize & ~0xfff)) SegSize = 4096; +// else if (SegSize > 16*1024*1024) SegSize = 16*1024*1024; + SegShft = 0; n = SegSize-1; + while(n) {SegShft++; n = n >> 1;} + SegSize = 1< 0 ? ParmV.CacheSize : 104857600)/SegSize; + if (SegCnt < minPag) SegCnt = minPag; + if (ParmV.Max2Cache < SegSize) maxCache = SegSize; + else maxCache = ParmV.Max2Cache/SegSize*SegSize; + SegFull = (Options & XrdRmc::isServer ? XrdRmcSlot::lenMask : SegSize); + +// Allocate the cache plus the cache hash table +// + Bytes = static_cast(SegSize)*SegCnt; + Base = (char *)mmap(0, Bytes + SegCnt*sizeof(int), PROT_READ|PROT_WRITE, + MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); + if (Base == MAP_FAILED) {rc = errno; return;} + Slash = (int *)(Base + Bytes); HNum = SegCnt/2*2-1; + +// Now allocate the actual slots. We add additional slots to map files. These +// do not have any memory backing but serve as anchors for memory mappings. +// + if (!(Slots = new XrdRmcSlot[SegCnt+maxFiles])) return; + XrdRmcSlot::Init(Slots, SegCnt); + +// Set pointers to be able to keep track of CacheIO objects and map them to +// CacheData objects. The hash table will be the first page of slot memory. +// + hTab = (int *)Base; + hMax = SegSize/sizeof(int); + sBeg = sFree = SegCnt; + sEnd = SegCnt + maxFiles; + +// Now iniialize the slots to be used for the CacheIO objects +// + for (n = sBeg; n < sEnd; n++) + {Slots[n].Own.Next = Slots[n].Own.Prev = n; + Slots[n].HLink = n+1; + } + Slots[sEnd-1].HLink = 0; + +// Setup the pre-readers if pre-read is enabled +// + if (Options & XrdRmc::canPreRead) + {pthread_t tid; + n = (Options & XrdRmc::isServer ? 9 : 3); + while(n--) + {if (XrdSysThread::Run(&tid, XrdRmcRealPRXeq, (void *)this, + 0, "Prereader")) break; + prNum++; + } + if (aprP && prNum) XrdRmcData::setAPR(aprDefault, *aprP, SegSize); + } + +// All done +// + rc = 0; +} + +/******************************************************************************/ +/* D e s t r u c t o r */ +/******************************************************************************/ + +XrdRmcReal::~XrdRmcReal() +{ +// Wait for all attachers to go away +// + CMutex.Lock(); + if (Attached) + {XrdSysSemaphore aZero(0); + AZero = &aZero; + CMutex.UnLock(); + aZero.Wait(); + CMutex.Lock(); + } + +// If any preread threads exist, then stop them now +// + prMutex.Lock(); + if (prNum) + {XrdSysSemaphore prDone(0); + prStop = &prDone; + prReady.Post(); + prMutex.UnLock(); + prDone.Wait(); + prMutex.Lock(); + } + +// Delete the slots +// + delete Slots; Slots = 0; + +// Unmap cache memory and associated hash table +// + if (Base != MAP_FAILED) + {munmap(Base, static_cast(SegSize)*SegCnt); + Base = (char *)(MAP_FAILED); + } + +// Release all locks, we are done +// + prMutex.UnLock(); + CMutex.UnLock(); +} + +/******************************************************************************/ +/* A t t a c h */ +/******************************************************************************/ + +XrdOucCacheIO *XrdRmcReal::Attach(XrdOucCacheIO *ioP, int Opts) +{ + static int Inst = 0; + XrdSysMutexHelper Monitor(CMutex); + XrdRmcData *dP; + int Cnt, Fnum = 0, theOpts = Opts & optRW; + +// Check if we are being deleted +// + if (AZero) {errno = ECANCELED; return ioP;} + +// Setup structured/unstructured option +// + if ((Opts & optFIS) || (Options & XrdRmc::isStructured)) theOpts |= optFIS; + +// Get an entry in the filename table. +// + if (!(Cnt = ioAdd(ioP, Fnum))) + {errno = EMFILE; + return ioP; + } + +// If this is the first addition then we need to get a new CacheData object. +// Otherwise, simply reuse the previous cache data object. +// + if (Cnt != 1) dP = Slots[Fnum].Status.Data; + else {long long vNum = static_cast(Fnum-SegCnt) << Shift + | (static_cast(Inst) << (Shift - 16)); + Inst = (Inst+1) & 0xffff; + if ((dP = new XrdRmcData(this, ioP, vNum, theOpts))) + {Attached++; Slots[Fnum].Status.Data = dP;} + } + +// Some debugging +// + if (Dbg) cerr <<"Cache: Attached " <Path() < 1) return 0; + +// We will be deleting the CramData object. So, we need to recycle its slots. +// + oP = &Slots[Fnum]; + while(oP->Own.Next != Fnum) + {sP = &Slots[oP->Own.Next]; + sP->Owner(Slots); + if (sP->Contents < 0 || sP->Status.LRU.Next < 0) Faults++; + else {sP->Hide(Slots, Slash, sP->Contents%HNum); + sP->Pull(Slots); + sP->unRef(Slots); + Free++; + } + } + +// Reduce attach count and check if the cache is being deleted +// + Attached--; + if (AZero && Attached <= 0) AZero->Post(); + +// Issue debugging message +// + if (Dbg) cerr <<"Cache: " <Path() <Contents != lAddr) {rAmt = -EIO; return 0;} + } else { + if (sP->Status.inUse < 0) sP->Status.inUse--; + else {sP->Pull(Slots); sP->Status.inUse = -1;} + } + rAmt = (sP->Count < 0 ? sP->Count & XrdRmcSlot::lenMask : SegSize); + if (sP->Count & XrdRmcSlot::isNew) + {noIO = -1; sP->Count &= ~XrdRmcSlot::isNew;} + if (Dbg > 2) cerr <<"Cache: Hit slot " <Status.inUse <(Slot)*SegSize); + } + +// Page is not here. If no allocation wanted or we cannot obtain a free slot +// return and indicate there is no associated cache page. +// + if (!ioP || !(Slot = Slots[Slots->Status.LRU.Next].Pull(Slots))) + {rAmt = -ENOMEM; return 0;} + +// Remove ownership over this slot and remove it from the hash table +// + sP = &Slots[Slot]; + if (sP->Contents >= 0) + {if (sP->Own.Next != Slot) sP->Owner(Slots); + sP->Hide(Slots, Slash, sP->Contents%HNum); + } + +// Read the data into the buffer +// + sP->Count |= XrdRmcSlot::inTrans; + sP->Status.waitQ = 0; + CMutex.UnLock(); + cBuff = Base+(static_cast(Slot)*SegSize); + rAmt = ioP->Read(cBuff, (lAddr & Strip) << SegShft, SegSize); + CMutex.Lock(); + +// Post anybody waiting for this slot. We hold the cache lock which will give us +// time to complete the slot definition before the waiting thread looks at it. +// + nUse = -1; + while((Waiter = sP->Status.waitQ)) + {sP->Status.waitQ = sP->Status.waitQ->Next; + sP->Status.waitQ->ioEnd->Post(); + nUse--; + } + +// If I/O succeeded, reinitialize the slot. Otherwise, return free it up +// + noIO = 0; + if (rAmt >= 0) + {sP->Contents = lAddr; + sP->HLink = Slash[segHash]; + Slash[segHash] = Slot; + Fnum = (lAddr >> Shift) + SegCnt; + Slots[Fnum].Owner(Slots, sP); + sP->Count = (rAmt == SegSize ? SegFull : rAmt|XrdRmcSlot::isShort); + sP->Status.inUse = nUse; + if (Dbg > 2) cerr <<"Cache: Miss slot " <Count & XrdRmcSlot::lenMask) <Path(), "reading", (lAddr & Strip) << SegShft, SegSize, rAmt); + cBuff = 0; + sP->Contents = -1; + sP->unRef(Slots); + } + +// Return the associated buffer or zero, as per above +// + return cBuff; +} + +/******************************************************************************/ +/* i o A d d */ +/******************************************************************************/ + +int XrdRmcReal::ioAdd(XrdOucCacheIO *KeyVal, int &iNum) +{ + int hip, phip, hent = ioEnt(KeyVal); + +// Look up the entry. If found, return it. +// + if ((hip = hTab[hent]) && (hip = ioLookup(phip, hip, KeyVal))) + {iNum = hip; return ++Slots[hip].Count;} + +// Add the entry +// + if ((hip = sFree)) + {sFree = Slots[sFree].HLink; + Slots[hip].File(KeyVal, hTab[hent]); + hTab[hent] = hip; + } + +// Return information to the caller +// + iNum = hip; + return (hip ? 1 : 0); +} + +/******************************************************************************/ +/* i o D e l */ +/******************************************************************************/ + +int XrdRmcReal::ioDel(XrdOucCacheIO *KeyVal, int &iNum) +{ + int cnt, hip, phip, hent = ioEnt(KeyVal); + +// Look up the entry. +// + if (!(hip = hTab[hent]) || !(hip = ioLookup(phip, hip, KeyVal))) return 0; + iNum = hip; + +// Delete the item, if need be, and return +// + cnt = --(Slots[hip].Count); + if (cnt <= 0) + {if (phip) Slots[phip].HLink = Slots[hip].HLink; + else hTab[hent] = Slots[hip].HLink; + Slots[hip].HLink = sFree; sFree = hip; + } + return (cnt < 0 ? 1 : cnt+1); +} + +/******************************************************************************/ +/* P r e R e a d */ +/******************************************************************************/ + +void XrdRmcReal::PreRead() +{ + prTask *prP; + +// Simply wait and dispatch elements +// + if (Dbg) cerr <<"Cache: preread thread started; now " <Next)) prLast = 0; + prMutex.UnLock(); + prP->Data->Preread(); + } else prMutex.UnLock(); + } + +// The cache is being deleted, wind down the prereads +// + prNum--; + if (prNum > 0) prReady.Post(); + else prStop->Post(); + if (Dbg) cerr <<"Cache: preread thread exited; left " <Next = prReq; prLast = prReq;} + else prLast = prFirst = prReq; + prReq->Next = 0; + +// Tell a pre-reader that something is ready +// + prReady.Post(); + prMutex.UnLock(); +} + +/******************************************************************************/ +/* R e f */ +/******************************************************************************/ + +int XrdRmcReal::Ref(char *Addr, int rAmt, int sFlags) +{ + XrdRmcSlot *sP = &Slots[(Addr-Base)>>SegShft]; + int eof = 0; + +// Indicate how much data was not yet referenced +// + CMutex.Lock(); + if (sP->Contents >= 0) + {if (sP->Count < 0) eof = 1; + sP->Status.inUse++; + if (sP->Status.inUse < 0) + {if (sFlags) sP->Count |= sFlags; + else if (!eof && (sP->Count -= rAmt) < 0) sP->Count = 0; + } else { + if (sFlags) {sP->Count |= sFlags; sP->reRef(Slots);} + else { if (sP->Count & XrdRmcSlot::isSUSE) + sP->unRef(Slots); + else if (eof || (sP->Count -= rAmt) > 0) sP->reRef(Slots); + else {sP->Count = SegSize/2; sP->unRef(Slots);} + } + } + } else eof = 1; + +// All done +// + if (Dbg > 2) cerr <<"Cache: Ref " <Contents <>SegShft) + <<" sz " <<(sP->Count & XrdRmcSlot::lenMask) + <<" uc " <Status.inUse <> Shift) + SegCnt; + +// We will be truncating CacheData pages. So, we need to recycle those slots. +// + oP = &Slots[Fnum]; sP = &Slots[oP->Own.Next]; + while(oP != sP) + {sNum = sP->Own.Next; + if (sP->Contents < lAddr) Left++; + else {sP->Owner(Slots); + sP->Hide(Slots, Slash, sP->Contents%HNum); + sP->Pull(Slots); + sP->unRef(Slots); + Free++; + } + sP = &Slots[sNum]; + } + +// Issue debugging message +// + if (Dbg) cerr <<"Cache: Trunc " <Path() <>SegShft]; + +// Check if we extended a short page +// + CMutex.Lock(); + if (sP->Count < 0) + {int theLen = sP->Count & XrdRmcSlot::lenMask; + if (wLen + wOff > theLen) + sP->Count = (wLen+wOff) | XrdRmcSlot::isShort; + } + +// Adjust the reference counter and if no references, place on the LRU chain +// + sP->Status.inUse++; + if (sP->Status.inUse >= 0) sP->reRef(Slots); + +// All done +// + if (Dbg > 2) cerr <<"Cache: Upd " <Contents <>SegShft) + <<" sz " <<(sP->Count & XrdRmcSlot::lenMask) + <<" uc " <Status.inUse <. */ +/* */ +/* The copyright holder's institutional names and contributor's names may not */ +/* be used to endorse or promote products derived from this software without */ +/* specific prior written permission of the institution or contributor. */ +/******************************************************************************/ + +#include "XrdRmc/XrdRmc.hh" +#include "XrdRmc/XrdRmcSlot.hh" +#include "XrdSys/XrdSysPthread.hh" + +/* This class defines an actual implementation of an XrdOucCache object. */ + +class XrdRmcReal : public XrdOucCache +{ +friend class XrdRmcData; +public: + +XrdOucCacheIO *Attach(XrdOucCacheIO *ioP, int Options=0); + + XrdRmcReal(int &rc, + XrdRmc::Parms &Parms, + XrdOucCacheIO::aprParms *aprP=0); + + ~XrdRmcReal(); + +void PreRead(); + +private: + +void eMsg(const char *Path, const char *What, long long xOff, + int xLen, int ec); +int Detach(XrdOucCacheIO *ioP); +char *Get(XrdOucCacheIO *ioP, long long lAddr, int &rGot, int &bIO); + +int ioAdd(XrdOucCacheIO *KeyVal, int &iNum); +int ioDel(XrdOucCacheIO *KeyVal, int &iNum); + +inline +int ioEnt(XrdOucCacheIO *kVal) + {union {short sV[4]; XrdOucCacheIO *pV;} Key = {{0,0,0,0}}; + Key.pV = kVal; + return ((Key.sV[0]^Key.sV[1]^Key.sV[2]^Key.sV[3])&0x7fff)%hMax; + } +inline +int ioLookup(int &pip, int hip, void *kval) + {pip = 0; + while(hip && kval != Slots[hip].Key) + {pip = hip; hip = Slots[hip].HLink;} + return hip; + } + +int Ref(char *Addr, int rAmt, int sFlags=0); +void Trunc(XrdOucCacheIO *ioP, long long lAddr); +void Upd(char *Addr, int wAmt, int wOff); + +static const long long Shift = 48; +static const long long Strip = 0x00000000ffffffffLL; // +static const long long MaxFO = 0x000007ffffffffffLL; // Min 4K page -> 8TB-1 + +XrdOucCacheIO::aprParms aprDefault; // Default automatic preread + +XrdSysMutex CMutex; +XrdRmcSlot *Slots; // 1-to-1 slot to memory map +int *Slash; // Slot hash table +char *Base; // Base of memory cache +long long HNum; +long long SegCnt; +long long SegSize; +long long OffMask; // SegSize - 1 +long long SegShft; // log2(SegSize) +int SegFull; // SegSize to mark +int maxCache; // Maximum read to cache +int maxFiles; // Maximum number of files to support +int Options; + +// The following supports CacheIO object tracking +// +int *hTab; // -> Hash Table +int hMax; // Number of entries in table +int sFree; // Index of free file slot +int sBeg; // Index of file slot array in slot table +int sEnd; // Last index + 1 + +// Various options +// +char Dbg; // Debug setting +char Lgs; // Log statistics + +// This is the attach/detach control area +// +XrdSysSemaphore *AZero; +int Attached; + +// This is the pre-read control area +// +struct prTask + {prTask *Next; + XrdRmcData *Data; + }; +void PreRead(XrdRmcReal::prTask *prReq); +prTask *prFirst; +prTask *prLast; +XrdSysMutex prMutex; +XrdSysSemaphore prReady; +XrdSysSemaphore *prStop; +int prNum; +}; +#endif diff --git a/src/XrdRmc/XrdRmcSlot.hh b/src/XrdRmc/XrdRmcSlot.hh new file mode 100644 index 00000000000..b02fc8ba5d8 --- /dev/null +++ b/src/XrdRmc/XrdRmcSlot.hh @@ -0,0 +1,154 @@ +#ifndef __XRDRMCSLOT_HH__ +#define __XRDRMCSLOT_HH__ +/******************************************************************************/ +/* */ +/* X r d R m c S l o t . h h */ +/* */ +/* (c) 2019 by the Board of Trustees of the Leland Stanford, Jr., University */ +/* All Rights Reserved */ +/* Produced by Andrew Hanushevsky for Stanford University under contract */ +/* DE-AC02-76-SFO0515 with the Department of Energy */ +/* */ +/* This file is part of the XRootD software suite. */ +/* */ +/* XRootD is free software: you can redistribute it and/or modify it under */ +/* the terms of the GNU Lesser General Public License as published by the */ +/* Free Software Foundation, either version 3 of the License, or (at your */ +/* option) any later version. */ +/* */ +/* XRootD is distributed in the hope that it will be useful, but WITHOUT */ +/* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ +/* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ +/* License for more details. */ +/* */ +/* You should have received a copy of the GNU Lesser General Public License */ +/* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ +/* COPYING (GPL license). If not, see . */ +/* */ +/* The copyright holder's institutional names and contributor's names may not */ +/* be used to endorse or promote products derived from this software without */ +/* specific prior written permission of the institution or contributor. */ +/******************************************************************************/ + +/* This class is used to support a memory cache used by an XrdOucCache actual + implementation. +*/ + +class XrdRmcData; +class XrdOucCacheIO; +class XrdSysSemaphore; + +class XrdRmcSlot +{ +public: + +inline void File(XrdOucCacheIO *kV, int you) + {Status.Data = 0; Key = kV; HLink = you; Count = 1;} + +static inline int Find(XrdRmcSlot *Base, long long What, int n) + {while(n && Base[n].Contents != What) n=Base[n].HLink; + return n; + } + +inline void Hide(XrdRmcSlot *Base, int *hTab, int hI) + {int j, Slot = this-Base; + if (hTab[hI] == Slot) hTab[hI] = HLink; + else if ((j = hTab[hI])) + {while((hI=Base[j].HLink) && hI != Slot) j=hI; + if (hI) Base[j].HLink = Base[hI].HLink; + } + Count = 0; Contents = -1; + } + +static void Init(XrdRmcSlot *Base, int Num) + {int i; + Base->Status.LRU.Next = Base->Status.LRU.Prev = 0; + Base->Own.Next = Base->Own.Prev = 0; + for (i = 1; i < Num; i++) + {Base[i].Status.LRU.Next = Base[i].Status.LRU.Prev = i; + Base[i].Own.Next = Base[i].Own.Prev = i; + Base->Push(Base, &Base[i]); + } + } + +inline int Pull(XrdRmcSlot *Base) + {Base[Status.LRU.Prev].Status.LRU.Next = Status.LRU.Next; + Base[Status.LRU.Next].Status.LRU.Prev = Status.LRU.Prev; + Status.LRU.Next = Status.LRU.Prev = this-Base; + return Status.LRU.Next; + } + +inline int Push(XrdRmcSlot *Base, XrdRmcSlot *sP) + {int UrNum = sP-Base, MyNum = this-Base; + sP->Status.LRU.Next = MyNum; + sP->Status.LRU.Prev = Status.LRU.Prev; + Base[Status.LRU.Prev].Status.LRU.Next = UrNum; + Status.LRU.Prev = UrNum; + return UrNum; + } + +inline void Owner(XrdRmcSlot *Base) + {Base[Own.Prev].Own.Next = Own.Next; + Base[Own.Next].Own.Prev = Own.Prev; + Own.Next = Own.Prev = this-Base; + } + +inline void Owner(XrdRmcSlot *Base, XrdRmcSlot *sP) + {int UrNum = sP-Base, MyNum = this-Base; + sP->Own.Next = MyNum; sP->Own.Prev = Own.Prev; + Base[Own.Prev].Own.Next = UrNum; Own.Prev = UrNum; + } + +inline void reRef(XrdRmcSlot *Base) + { Status.LRU.Prev = Base->Status.LRU.Prev; + Base[ Status.LRU.Prev].Status.LRU.Next = this-Base; + Base->Status.LRU.Prev = this-Base; + Status.LRU.Next = 0; + } + +inline void unRef(XrdRmcSlot *Base) + { Status.LRU.Next = Base->Status.LRU.Next; + Base [Status.LRU.Next].Status.LRU.Prev = this-Base; + Base->Status.LRU.Next = this-Base; + Status.LRU.Prev = 0; + } + +struct SlotList + { + int Next; + int Prev; + }; + +struct ioQ + {ioQ *Next; + XrdSysSemaphore *ioEnd; + ioQ(ioQ *First, XrdSysSemaphore *ioW) + : Next(First), ioEnd(ioW) {} + }; + +union SlotState + {struct ioQ *waitQ; + XrdRmcData *Data; + struct SlotList LRU; + int inUse; + }; + +union {long long Contents; + XrdOucCacheIO *Key; + }; +SlotState Status; +SlotList Own; +int HLink; +int Count; + +static const int lenMask = 0x01ffffff; // Mask to get true value in Count +static const int isShort = 0x80000000; // Short page, Count & lenMask == size +static const int inTrans = 0x40000000; // Segment is in transit +static const int isSUSE = 0x20000000; // Segment is single use +static const int isNew = 0x10000000; // Segment is new (not yet referenced) + + XrdRmcSlot() : Contents(-1), HLink(0), Count(0) {} + + ~XrdRmcSlot() {} +}; +#endif diff --git a/src/XrdSfs/XrdSfsInterface.hh b/src/XrdSfs/XrdSfsInterface.hh index 926d2d89052..044619effa6 100644 --- a/src/XrdSfs/XrdSfsInterface.hh +++ b/src/XrdSfs/XrdSfsInterface.hh @@ -1263,23 +1263,12 @@ virtual ~XrdSfsFileSystem() {} @param Logger - The message logging object to be used for messages. @param configFN - pointer to the path of the configuration file. If nil there is no configuration file. + @param envP - Pointer to the environment containing implementation + specific information. @return Pointer to the file system object to be used or nil if an error occurred. - extern "C" - {XrdSfsFileSystem *XrdSfsGetFileSystem(XrdSfsFileSystem *nativeFS, - XrdSysLogger *Logger, - const char *configFn); - } - - An alternate entry point may be defined in lieu of the previous entry point. - This normally identified by a version option in the configuration file (e.g. - xrootd.fslib -2 ). It differs in that an extra parameter is passed: - - @param envP - Pointer to the environment containing implementation - specific information. - extern "C" {XrdSfsFileSystem *XrdSfsGetFileSystem2(XrdSfsFileSystem *nativeFS, XrdSysLogger *Logger, @@ -1288,17 +1277,27 @@ virtual ~XrdSfsFileSystem() {} } */ -typedef XrdSfsFileSystem *(*XrdSfsFileSystem_t) (XrdSfsFileSystem *nativeFS, - XrdSysLogger *Logger, - const char *configFn); - typedef XrdSfsFileSystem *(*XrdSfsFileSystem2_t)(XrdSfsFileSystem *nativeFS, XrdSysLogger *Logger, const char *configFn, XrdOucEnv *envP); //----------------------------------------------------------------------------- - +/*! The old-style entry-point is still supported as a fallback. Should the + version '2' entry point is not found, the system attempts to use the + version '1' entry point. + + extern "C" + {XrdSfsFileSystem *XrdSfsGetFileSystem(XrdSfsFileSystem *nativeFS, + XrdSysLogger *Logger, + const char *configFn); + } +*/ + +typedef XrdSfsFileSystem *(*XrdSfsFileSystem_t) (XrdSfsFileSystem *nativeFS, + XrdSysLogger *Logger, + const char *configFn); + //------------------------------------------------------------------------------ /*! Specify the compilation version. diff --git a/src/XrdUtils.cmake b/src/XrdUtils.cmake index d559dd7ddf1..e7d46ec163d 100644 --- a/src/XrdUtils.cmake +++ b/src/XrdUtils.cmake @@ -75,12 +75,6 @@ add_library( XrdOuc/XrdOucBackTrace.cc XrdOuc/XrdOucBackTrace.hh XrdOuc/XrdOucBuffer.cc XrdOuc/XrdOucBuffer.hh XrdOuc/XrdOucCache.hh - XrdOuc/XrdOucCache2.hh - XrdOuc/XrdOucCacheData.cc XrdOuc/XrdOucCacheData.hh - XrdOuc/XrdOucCacheDram.cc XrdOuc/XrdOucCacheDram.hh - XrdOuc/XrdOucCacheCM.hh - XrdOuc/XrdOucCacheReal.cc XrdOuc/XrdOucCacheReal.hh - XrdOuc/XrdOucCacheSlot.hh XrdOuc/XrdOucCacheStats.hh XrdOuc/XrdOucCallBack.cc XrdOuc/XrdOucCallBack.hh XrdOuc/XrdOucCRC.cc XrdOuc/XrdOucCRC.hh @@ -203,6 +197,14 @@ add_library( XrdCks/XrdCks.hh XrdCks/XrdCksXAttr.hh + #----------------------------------------------------------------------------- + # XrdRmc + #----------------------------------------------------------------------------- + XrdRmc/XrdRmc.cc XrdRmc/XrdRmc.hh + XrdRmc/XrdRmcData.cc XrdRmc/XrdRmcData.hh + XrdRmc/XrdRmcReal.cc XrdRmc/XrdRmcReal.hh + XrdRmc/XrdRmcSlot.hh + #----------------------------------------------------------------------------- # XrdSec #----------------------------------------------------------------------------- diff --git a/src/XrdVersionPlugin.hh b/src/XrdVersionPlugin.hh index 985f135a1e0..9a6baab6246 100644 --- a/src/XrdVersionPlugin.hh +++ b/src/XrdVersionPlugin.hh @@ -176,7 +176,9 @@ "libXrdCryptossl.so", \ "libXrdFileCache.so", \ "libXrdHttp.so", \ + "libXrdHttpTPC.so", \ "libXrdOssSIgpfsT.so", \ + "libXrdN2No2p.so", \ "libXrdPss.so", \ "libXrdSec.so", \ "libXrdSecgsi.so", \ @@ -186,6 +188,9 @@ "libXrdSecpwd.so", \ "libXrdSecsss.so", \ "libXrdSecunix.so", \ + "libXrdSsi.so", \ + "libXrdSsLog.so", \ + "libXrdThrottle.so", \ "libXrdXrootd.so", \ 0} #endif