diff --git a/src/XrdFileCache.cmake b/src/XrdFileCache.cmake index 46ed47193c9..f241adc7fec 100644 --- a/src/XrdFileCache.cmake +++ b/src/XrdFileCache.cmake @@ -18,9 +18,11 @@ add_library( MODULE XrdFileCache/XrdFileCache.cc XrdFileCache/XrdFileCache.hh XrdFileCache/XrdFileCacheFactory.cc XrdFileCache/XrdFileCacheFactory.hh - XrdFileCache/XrdFileCachePrefetch.cc XrdFileCache/XrdFileCachePrefetch.hh + XrdFileCache/XrdFileCacheFile.cc XrdFileCache/XrdFileCacheFile.hh + XrdFileCache/XrdFileCacheVRead.cc XrdFileCache/XrdFileCacheStats.hh XrdFileCache/XrdFileCacheInfo.cc XrdFileCache/XrdFileCacheInfo.hh + XrdFileCache/XrdFileCacheIO.hh XrdFileCache/XrdFileCacheIOEntireFile.cc XrdFileCache/XrdFileCacheIOEntireFile.hh XrdFileCache/XrdFileCacheIOFileBlock.cc XrdFileCache/XrdFileCacheIOFileBlock.hh XrdFileCache/XrdFileCacheDecision.hh) diff --git a/src/XrdFileCache/XrdFileCache.cc b/src/XrdFileCache/XrdFileCache.cc index 95be0e484f3..a103b032176 100644 --- a/src/XrdFileCache/XrdFileCache.cc +++ b/src/XrdFileCache/XrdFileCache.cc @@ -18,25 +18,24 @@ #include #include +#include #include #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClURL.hh" #include "XrdSys/XrdSysPthread.hh" +#include "XrdSys/XrdSysTimer.hh" #include "XrdOss/XrdOss.hh" #include "XrdOuc/XrdOucEnv.hh" #include "XrdFileCache.hh" -#include "XrdFileCachePrefetch.hh" #include "XrdFileCacheIOEntireFile.hh" #include "XrdFileCacheIOFileBlock.hh" #include "XrdFileCacheFactory.hh" -#include "XrdFileCachePrefetch.hh" -XrdFileCache::Cache::WriteQ XrdFileCache::Cache::s_writeQ; - using namespace XrdFileCache; + void *ProcessWriteTaskThread(void* c) { Cache *cache = static_cast(c); @@ -44,13 +43,27 @@ void *ProcessWriteTaskThread(void* c) return NULL; } -Cache::Cache(XrdOucCacheStats & stats) - : m_attached(0), - m_stats(stats) +void *PrefetchThread(void* ptr) +{ + Cache* cache = static_cast(ptr); + cache->Prefetch(); + return NULL; +} +//______________________________________________________________________________ + + +Cache::Cache(XrdOucCacheStats & stats) : XrdOucCache(), + m_prefetch_condVar(0), + m_stats(stats), + m_RAMblocks_used(0) { - pthread_t tid; - XrdSysThread::Run(&tid, ProcessWriteTaskThread, (void*)this, 0, "XrdFileCache WriteTasks "); + pthread_t tid1; + XrdSysThread::Run(&tid1, ProcessWriteTaskThread, (void*)this, 0, "XrdFileCache WriteTasks "); + + pthread_t tid2; + XrdSysThread::Run(&tid2, PrefetchThread, (void*)this, 0, "XrdFileCache Prefetch "); } + //______________________________________________________________________________ XrdOucCacheIO *Cache::Attach(XrdOucCacheIO *io, int Options) @@ -58,17 +71,12 @@ XrdOucCacheIO *Cache::Attach(XrdOucCacheIO *io, int Options) if (Factory::GetInstance().Decide(io)) { clLog()->Info(XrdCl::AppMsg, "Cache::Attach() %s", io->Path()); - { - XrdSysMutexHelper lock(&m_io_mutex); - m_attached++; - } IO* cio; if (Factory::GetInstance().RefConfiguration().m_hdfsmode) cio = new IOFileBlock(*io, m_stats, *this); else cio = new IOEntireFile(*io, m_stats, *this); - cio->StartPrefetch(); return cio; } else @@ -82,65 +90,57 @@ XrdOucCacheIO *Cache::Attach(XrdOucCacheIO *io, int Options) int Cache::isAttached() { - XrdSysMutexHelper lock(&m_io_mutex); - return m_attached; + // virutal function of XrdOucCache, don't see it used in pfc or posix layer + return true; } void Cache::Detach(XrdOucCacheIO* io) { clLog()->Info(XrdCl::AppMsg, "Cache::Detach() %s", io->Path()); - { - XrdSysMutexHelper lock(&m_io_mutex); - m_attached--; - } delete io; } - -//______________________________________________________________________________ - - -void Cache::getFilePathFromURL(const char* iUrl, std::string &result) const -{ - XrdCl::URL url(iUrl); - result = Factory::GetInstance().RefConfiguration().m_cache_dir + url.GetPath(); -} - //______________________________________________________________________________ bool Cache::HaveFreeWritingSlots() { - const static size_t maxWriteWaits=500; - return s_writeQ.size < maxWriteWaits; + const static size_t maxWriteWaits=100000; + if ( s_writeQ.size < maxWriteWaits) { + return true; + } + else { + XrdCl::DefaultEnv::GetLog()->Info(XrdCl::AppMsg, "Cache::HaveFreeWritingSlots() negative", s_writeQ.size); + return false; + } } - - //______________________________________________________________________________ void -Cache::AddWriteTask(Prefetch* p, int ri, size_t s, bool fromRead) +Cache::AddWriteTask(Block* b, bool fromRead) { - XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::AddWriteTask() wqsize = %d, bi=%d", s_writeQ.size, ri); + XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::AddWriteTask() bOff=%ld", b->m_offset); s_writeQ.condVar.Lock(); if (fromRead) - s_writeQ.queue.push_back(WriteTask(p, ri, s)); + s_writeQ.queue.push_back(b); else - s_writeQ.queue.push_front(WriteTask(p, ri, s)); + s_writeQ.queue.push_front(b); // AMT should this not be the opposite s_writeQ.size++; s_writeQ.condVar.Signal(); s_writeQ.condVar.UnLock(); } //______________________________________________________________________________ -void Cache::RemoveWriteQEntriesFor(Prefetch *p) +void Cache::RemoveWriteQEntriesFor(File *iFile) { s_writeQ.condVar.Lock(); - std::list::iterator i = s_writeQ.queue.begin(); + std::list::iterator i = s_writeQ.queue.begin(); while (i != s_writeQ.queue.end()) { - if (i->prefetch == p) + if ((*i)->m_file == iFile) { - std::list::iterator j = i++; - j->prefetch->DecRamBlockRefCount(j->ramBlockIdx); + + XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::Remove entries for %p path %s", (void*)(*i), iFile->lPath()); + std::list::iterator j = i++; + iFile->BlockRemovedFromWriteQ(*j); s_writeQ.queue.erase(j); --s_writeQ.size; } @@ -163,12 +163,127 @@ Cache::ProcessWriteTasks() { s_writeQ.condVar.Wait(); } - WriteTask t = s_writeQ.queue.front(); + Block* block = s_writeQ.queue.front(); // AMT should not be back ??? s_writeQ.queue.pop_front(); s_writeQ.size--; + XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::ProcessWriteTasks for %p path %s", (void*)(block), block->m_file->lPath()); s_writeQ.condVar.UnLock(); - t.prefetch->WriteBlockToDisk(t.ramBlockIdx, t.size); - t.prefetch->DecRamBlockRefCount(t.ramBlockIdx); + block->m_file->WriteBlockToDisk(block); } } + +//______________________________________________________________________________ + +bool +Cache::RequestRAMBlock() +{ + XrdSysMutexHelper lock(&m_RAMblock_mutex); + if ( m_RAMblocks_used < Factory::GetInstance().RefConfiguration().m_NRamBuffers ) + { + m_RAMblocks_used++; + return true; + } + return false; +} + +void +Cache::RAMBlockReleased() +{ + XrdSysMutexHelper lock(&m_RAMblock_mutex); + m_RAMblocks_used--; +} + + +//============================================================================== +//======================= PREFETCH =================================== +//============================================================================== +/* +namespace { +struct prefetch_less_than +{ + inline bool operator() (const File* struct1, const File* struct2) + { + return (struct1->GetPrefetchScore() < struct2->GetPrefetchScore()); + } +}myobject; +}*/ +//______________________________________________________________________________ + +void +Cache::RegisterPrefetchFile(File* file) +{ + // called from File::Open() + + if (Factory::GetInstance().RefConfiguration().m_prefetch) + { + + XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::Register new file BEGIN"); + m_prefetch_condVar.Lock(); + m_files.push_back(file); + m_prefetch_condVar.Signal(); + m_prefetch_condVar.UnLock(); + XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::Register new file End"); + } +} + +//______________________________________________________________________________ + +void +Cache::DeRegisterPrefetchFile(File* file) +{ + // called from last line File::InitiateClose() + + m_prefetch_condVar.Lock(); + for (FileList::iterator it = m_files.begin(); it != m_files.end(); ++it) { + if (*it == file) { + m_files.erase(it); + break; + } + } + m_prefetch_condVar.UnLock(); +} +//______________________________________________________________________________ + +File* +Cache::GetNextFileToPrefetch() +{ + m_prefetch_condVar.Lock(); + if (m_files.empty()) { + m_prefetch_condVar.Wait(); + } + + // std::sort(m_files.begin(), m_files.end(), myobject); + + size_t l = m_files.size(); + int idx = rand() % l; + File* f = m_files[idx]; + f->MarkPrefetch(); + m_prefetch_condVar.UnLock(); + return f; +} + +//______________________________________________________________________________ + + +void +Cache::Prefetch() +{ + const static int limitRAM= Factory::GetInstance().RefConfiguration().m_NRamBuffers * 0.7; + + XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg, "Cache::Prefetch thread start"); + + while (true) { + m_RAMblock_mutex.Lock(); + bool doPrefetch = (m_RAMblocks_used < limitRAM && HaveFreeWritingSlots()); + m_RAMblock_mutex.UnLock(); + + if (doPrefetch) { + File* f = GetNextFileToPrefetch(); + f->Prefetch(); + } + else { + XrdSysTimer::Wait(5); + } + } +} diff --git a/src/XrdFileCache/XrdFileCache.hh b/src/XrdFileCache/XrdFileCache.hh index 4d48a4e0148..af616ec0c75 100644 --- a/src/XrdFileCache/XrdFileCache.hh +++ b/src/XrdFileCache/XrdFileCache.hh @@ -23,12 +23,14 @@ #include "XrdSys/XrdSysPthread.hh" #include "XrdOuc/XrdOucCache.hh" #include "XrdCl/XrdClDefaultEnv.hh" +#include "XrdFileCacheFile.hh" namespace XrdCl { class Log; } namespace XrdFileCache { -class Prefetch; +class File; +class IO; } namespace XrdFileCache @@ -38,9 +40,6 @@ namespace XrdFileCache //---------------------------------------------------------------------------- class Cache : public XrdOucCache { - friend class IOEntireFile; - friend class IOFileBlock; - public: //--------------------------------------------------------------------- //! Constructor @@ -67,94 +66,64 @@ namespace XrdFileCache //--------------------------------------------------------------------- //! Add downloaded block in write queue. //--------------------------------------------------------------------- - static void AddWriteTask(Prefetch* p, int ramBlockidx, size_t size, bool fromRead); + void AddWriteTask(Block* b, bool from_read); //--------------------------------------------------------------------- //! Check write queue size is not over limit. //--------------------------------------------------------------------- - static bool HaveFreeWritingSlots(); + bool HaveFreeWritingSlots(); //--------------------------------------------------------------------- //! \brief Remove blocks from write queue which belong to given prefetch. - //! This method is used at the time of Prefetch destruction. + //! This method is used at the time of File destruction. //--------------------------------------------------------------------- - static void RemoveWriteQEntriesFor(Prefetch *p); + void RemoveWriteQEntriesFor(File *f); //--------------------------------------------------------------------- //! Separate task which writes blocks from ram to disk. //--------------------------------------------------------------------- - static void ProcessWriteTasks(); + void ProcessWriteTasks(); + + bool RequestRAMBlock(); + + void RAMBlockReleased(); + + void RegisterPrefetchFile(File*); + void DeRegisterPrefetchFile(File*); + + File* GetNextFileToPrefetch(); + + void Prefetch(); - private: //! Decrease attached count. Called from IO::Detach(). void Detach(XrdOucCacheIO *); - //! Transfor URL to path on local disk. - void getFilePathFromURL(const char* url, std::string& res) const; + private: //! Short log alias. XrdCl::Log* clLog() const { return XrdCl::DefaultEnv::GetLog(); } - XrdSysMutex m_io_mutex; //!< central lock for this class - unsigned int m_attached; //!< number of attached IO objects + XrdSysCondVar m_prefetch_condVar; //!< central lock for this class XrdOucCacheStats &m_stats; //!< global cache usage statistics - struct WriteTask - { - Prefetch* prefetch; //!< object queued for writing - int ramBlockIdx; //!< in memory cache index - size_t size; //!< write size -- block size except in case this is the end file block - WriteTask(Prefetch* p, int ri, size_t s):prefetch(p), ramBlockIdx(ri), size(s){} - }; + XrdSysMutex m_RAMblock_mutex; //!< central lock for this class + int m_RAMblocks_used; struct WriteQ { WriteQ() : condVar(0), size(0) {} XrdSysCondVar condVar; //!< write list condVar size_t size; //!< cache size of a container - std::list queue; //!< container + std::list queue; //!< container }; - static WriteQ s_writeQ; + WriteQ s_writeQ; + // prefetching + typedef std::vector FileList; + FileList m_files; }; - //---------------------------------------------------------------------------- - //! Base cache-io class that implements XrdOucCacheIO abstract methods. - //---------------------------------------------------------------------------- - class IO : public XrdOucCacheIO - { - friend class Prefetch; - - public: - IO (XrdOucCacheIO &io, XrdOucCacheStats &stats, Cache &cache) : - m_io(io), m_statsGlobal(stats), m_cache(cache) {} - - //! Original data source. - virtual XrdOucCacheIO *Base() { return &m_io; } - - //! Original data source URL. - virtual long long FSize() { return m_io.FSize(); } - - //! Original data source URL. - virtual const char *Path() { return m_io.Path(); } - - virtual int Sync() { return 0; } - - virtual int Trunc(long long Offset) { errno = ENOTSUP; return -1; } - - virtual int Write(char *Buffer, long long Offset, int Length) - { errno = ENOTSUP; return -1; } - - virtual void StartPrefetch() {} - - protected: - XrdCl::Log* clLog() const { return XrdCl::DefaultEnv::GetLog(); } - - XrdOucCacheIO &m_io; //!< original data source - XrdOucCacheStats &m_statsGlobal; //!< reference to Cache statistics - Cache &m_cache; //!< reference to Cache needed in detach - }; } #endif diff --git a/src/XrdFileCache/XrdFileCacheConfiguration.cc b/src/XrdFileCache/XrdFileCacheConfiguration.cc new file mode 100644 index 00000000000..7f45d454a83 --- /dev/null +++ b/src/XrdFileCache/XrdFileCacheConfiguration.cc @@ -0,0 +1,299 @@ +#include "XrdFileCache.hh" + +#include "XrdOss/XrdOss.hh" +#include "XrdOss/XrdOssCache.hh" + +#include "XrdOuc/XrdOucEnv.hh" +#include "XrdOuc/XrdOucUtils.hh" +#include "XrdOuc/XrdOucStream.hh" +#include "XrdOuc/XrdOucPinLoader.hh" +#include "XrdOuc/XrdOuca2x.hh" + +#include "XrdOfs/XrdOfsConfigPI.hh" +#include "XrdVersion.hh" + +#include +using namespace XrdFileCache; + +XrdVERSIONINFO(XrdOucGetCache, XrdFileCache); +/* Function: xdlib + + Purpose: To parse the directive: decisionlib [] + + the path of the decision library to be used. + optional parameters to be passed. + + + Output: true upon success or false upon failure. + */ +bool Cache::xdlib(XrdOucStream &Config) +{ + const char* val; + + std::string libp; + if (!(val = Config.GetWord()) || !val[0]) + { + clLog()->Info(XrdCl::AppMsg, " Cache::Config() decisionlib not specified; always caching files"); + return true; + } + else + { + libp = val; + } + + const char* params; + params = (val[0]) ? Config.GetWord() : 0; + + XrdOucPinLoader* myLib = new XrdOucPinLoader(&m_log, 0, "decisionlib", + libp.c_str()); + + Decision *(*ep)(XrdSysError&); + ep = (Decision *(*)(XrdSysError&))myLib->Resolve("XrdFileCacheGetDecision"); + if (!ep) {myLib->Unload(true); return false;} + + Decision * d = ep(m_log); + if (!d) + { + clLog()->Error(XrdCl::AppMsg, "Cache::Config() decisionlib was not able to create a decision object"); + return false; + } + if (params) + d->ConfigDecision(params); + + m_decisionpoints.push_back(d); + clLog()->Info(XrdCl::AppMsg, "Cache::Config() successfully created decision lib from %s", libp.c_str()); + return true; +} + +//______________________________________________________________________________ + +bool Cache::Config(XrdSysLogger *logger, const char *config_filename, const char *parameters) +{ + m_log.logger(logger); + + const char * cache_env; + if (!(cache_env = getenv("XRDPOSIX_CACHE")) || !*cache_env) + XrdOucEnv::Export("XRDPOSIX_CACHE", "mode=s&optwr=0"); + + XrdOucEnv myEnv; + XrdOucStream Config(&m_log, getenv("XRDINSTANCE"), &myEnv, "=====> "); + + if (!config_filename || !*config_filename) + { + clLog()->Warning(XrdCl::AppMsg, "Cache::Config() configuration file not specified."); + return false; + } + + int fd; + if ( (fd = open(config_filename, O_RDONLY, 0)) < 0) + { + clLog()->Error(XrdCl::AppMsg, "Cache::Config() can't open configuration file %s", config_filename); + return false; + } + + Config.Attach(fd); + + // Obtain plugin configurator + XrdOfsConfigPI *ofsCfg = XrdOfsConfigPI::New(config_filename,&Config,&m_log, + &XrdVERSIONINFOVAR(XrdOucGetCache)); + if (!ofsCfg) return false; + + + if (ofsCfg->Load(XrdOfsConfigPI::theOssLib)) { + ofsCfg->Plugin(m_output_fs); + XrdOssCache_FS* ocfs = XrdOssCache::Find("public"); + ocfs->Add(m_configuration.m_cache_dir.c_str()); + } + else + { + clLog()->Error(XrdCl::AppMsg, "Cache::Config() Unable to create an OSS object"); + m_output_fs = 0; + return false; + } + + + // Actual parsing of the config file. + bool retval = true; + char *var; + while((var = Config.GetMyFirstWord())) + { + if (!strcmp(var,"pfc.osslib")) + { + ofsCfg->Parse(XrdOfsConfigPI::theOssLib); + } + else if (!strcmp(var,"pfc.decisionlib")) + { + xdlib(Config); + } + else if (!strncmp(var,"pfc.", 4)) + { + retval = ConfigParameters(std::string(var+4), Config); + } + + if (!retval) + { + retval = false; + clLog()->Error(XrdCl::AppMsg, "Cache::Config() error in parsing"); + break; + } + + } + + Config.Close(); + // sets default value for disk usage + if (m_configuration.m_diskUsageLWM < 0 || m_configuration.m_diskUsageHWM < 0) + { + XrdOssVSInfo sP; + if (m_output_fs->StatVS(&sP, "public", 1) >= 0) { + m_configuration.m_diskUsageLWM = static_cast(0.90 * sP.Total + 0.5); + m_configuration.m_diskUsageHWM = static_cast(0.95 * sP.Total + 0.5); + clLog()->Debug(XrdCl::AppMsg, "Default disk usage [%lld, %lld]", m_configuration.m_diskUsageLWM, m_configuration.m_diskUsageHWM); + } + } + + if (retval) + { + int loff = 0; + char buff[2048]; + loff = snprintf(buff, sizeof(buff), "result\n" + "\tpfc.cachedir %s\n" + "\tpfc.blocksize %lld\n" + "\tpfc.prefetch %d\n" + "\tpfc.nram %d\n\n", + m_configuration.m_cache_dir.c_str() , + m_configuration.m_bufferSize, + m_configuration.m_prefetch, // AMT not sure what parsing should be + m_configuration.m_NRamBuffers ); + + if (m_configuration.m_hdfsmode) + { + char buff2[512]; + snprintf(buff2, sizeof(buff2), "\tpfc.hdfsmode hdfsbsize %lld\n", m_configuration.m_hdfsbsize); + loff += snprintf(&buff[loff], strlen(buff2), "%s", buff2); + } + + char unameBuff[256]; + if (m_configuration.m_username.empty()) { + XrdOucUtils::UserName(getuid(), unameBuff, sizeof(unameBuff)); + m_configuration.m_username = unameBuff; + } + else { + snprintf(unameBuff, sizeof(unameBuff), "\tpfc.user %s \n", m_configuration.m_username.c_str()); + loff += snprintf(&buff[loff], strlen(unameBuff), "%s", unameBuff); + } + + m_log.Emsg("Config", buff); + } + + m_log.Emsg("Config", "Configuration = ", retval ? "Success" : "Fail"); + + if (ofsCfg) delete ofsCfg; + return retval; +} + +//______________________________________________________________________________ + + +bool Cache::ConfigParameters(std::string part, XrdOucStream& config ) +{ + printf("part %s \n", part.c_str()); + XrdSysError err(0, ""); + if ( part == "user" ) + { + m_configuration.m_username = config.GetWord(); + } + else if ( part == "cachedir" ) + { + m_configuration.m_cache_dir = config.GetWord(); + } + else if ( part == "diskusage" ) + { + std::string minV = config.GetWord(); + std::string maxV = config.GetWord(); + if (!minV.empty() && !maxV.empty()) { + XrdOssVSInfo sP; + if (m_output_fs->StatVS(&sP, "public", 1) >= 0) + { + if (::isalpha(*(minV.rbegin())) && ::isalpha(*(minV.rbegin()))) { + if ( XrdOuca2x::a2sz(m_log, "Error getting disk usage low watermark", minV.c_str(), &m_configuration.m_diskUsageLWM, 0, sP.Total) + || XrdOuca2x::a2sz(m_log, "Error getting disk usage high watermark", maxV.c_str(), &m_configuration.m_diskUsageHWM, 0, sP.Total)) + { + return false; + } + } + else + { + char* eP; + errno = 0; + float lwmf = strtod(minV.c_str(), &eP); + if (errno || eP == minV.c_str()) { + m_log.Emsg("Cache::ConfigParameters() error parsing diskusage parameter ", minV.c_str()); + return false; + } + float hwmf = strtod(maxV.c_str(), &eP); + if (errno || eP == maxV.c_str()) { + m_log.Emsg("Cache::ConfigParameters() error parsing diskusage parameter ", maxV.c_str()); + return false; + } + + m_configuration.m_diskUsageLWM = static_cast(sP.Total * lwmf + 0.5); + m_configuration.m_diskUsageHWM = static_cast(sP.Total * hwmf + 0.5); + } + } + } + } + else if ( part == "blocksize" ) + { + long long minBSize = 64 * 1024; + long long maxBSize = 16 * 1024 * 1024; + if ( XrdOuca2x::a2sz(m_log, "get block size", config.GetWord(), &m_configuration.m_bufferSize, minBSize, maxBSize)) + { + return false; + } + } + else if (part == "prefetch" ) + { + int p = ::atoi(config.GetWord()); + if (p > 0) { + printf("prefetch enabled, max blocks per file=%d\n", p); + m_configuration.m_prefetch = true; + m_configuration.m_prefetch_max_blocks = p; + } else { + m_configuration.m_prefetch = false; + } + } + else if (part == "nram" ) + { + m_configuration.m_NRamBuffers = ::atoi(config.GetWord()); + } + else if ( part == "hdfsmode" ) + { + m_configuration.m_hdfsmode = true; + + const char* params = config.GetWord(); + if (params) { + if (!strncmp("hdfsbsize", params, 9)) { + long long minBlSize = 128 * 1024; + long long maxBlSize = 1024 * 1024 * 1024; + params = config.GetWord(); + if ( XrdOuca2x::a2sz(m_log, "Error getting file fragment size", params, &m_configuration.m_hdfsbsize, minBlSize, maxBlSize)) + { + return false; + } + } + else { + m_log.Emsg("Config", "Error setting the fragment size parameter name"); + return false; + } + } + } + else + { + m_log.Emsg("Cache::ConfigParameters() unmatched pfc parameter", part.c_str()); + return false; + } + + assert ( config.GetWord() == 0 && "Cache::ConfigParameters() lost argument"); + + return true; +} diff --git a/src/XrdFileCache/XrdFileCacheFactory.cc b/src/XrdFileCache/XrdFileCacheFactory.cc deleted file mode 100644 index 93b63762395..00000000000 --- a/src/XrdFileCache/XrdFileCacheFactory.cc +++ /dev/null @@ -1,607 +0,0 @@ -//---------------------------------------------------------------------------------- -// Copyright (c) 2014 by Board of Trustees of the Leland Stanford, Jr., University -// Author: Alja Mrak-Tadel, Matevz Tadel, Brian Bockelman -//---------------------------------------------------------------------------------- -// XRootD is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// XRootD is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with XRootD. If not, see . -//---------------------------------------------------------------------------------- - -#include -#include -#include -#include - - -#include - - -#include "XrdSys/XrdSysPthread.hh" -#include "XrdSys/XrdSysError.hh" -#include "XrdSys/XrdSysLogger.hh" -#include "XrdOuc/XrdOucEnv.hh" -#include "XrdOuc/XrdOucUtils.hh" -#include "XrdOss/XrdOssCache.hh" -#include "XrdOuc/XrdOucPinLoader.hh" -#include "XrdOuc/XrdOucStream.hh" -#include "XrdOuc/XrdOuca2x.hh" -#include "XrdOfs/XrdOfsConfigPI.hh" -#include "XrdOss/XrdOss.hh" -#include "XrdVersion.hh" -#include "XrdPosix/XrdPosixXrootd.hh" -#include "XrdCl/XrdClConstants.hh" - -#include "XrdFileCache.hh" -#include "XrdFileCacheFactory.hh" -#include "XrdFileCachePrefetch.hh" - - -using namespace XrdFileCache; - -XrdVERSIONINFO(XrdOucGetCache, XrdFileCache); - - -Factory * Factory::m_factory = NULL; - -void *CacheDirCleanupThread(void* cache_void) -{ - Factory::GetInstance().CacheDirCleanup(); - return NULL; -} - - -Factory::Factory() - : m_log(0, "XrdFileCache_") -{} - -extern "C" -{ -XrdOucCache *XrdOucGetCache(XrdSysLogger *logger, - const char *config_filename, - const char *parameters) -{ - XrdSysError err(0, ""); - err.logger(logger); - err.Emsg("Retrieve", "Retrieving a caching proxy factory."); - Factory &factory = Factory::GetInstance(); - if (!factory.Config(logger, config_filename, parameters)) - { - err.Emsg("Retrieve", "Error - unable to create a factory."); - return NULL; - } - err.Emsg("Retrieve", "Success - returning a factory."); - - - pthread_t tid; - XrdSysThread::Run(&tid, CacheDirCleanupThread, NULL, 0, "XrdFileCache CacheDirCleanup"); - return &factory; -} -} - -Factory &Factory::GetInstance() -{ - if (m_factory == NULL) - m_factory = new Factory(); - return *m_factory; -} - -XrdOucCache *Factory::Create(Parms & parms, XrdOucCacheIO::aprParms * prParms) -{ - clLog()->Info(XrdCl::AppMsg, "Factory::Create() new cache object"); - return new Cache(m_stats); -} - - -/* Function: xdlib - - Purpose: To parse the directive: decisionlib [] - - the path of the decision library to be used. - optional parameters to be passed. - - - Output: true upon success or false upon failure. - */ -bool Factory::xdlib(XrdOucStream &Config) -{ - const char* val; - - std::string libp; - if (!(val = Config.GetWord()) || !val[0]) - { - clLog()->Info(XrdCl::AppMsg, " Factory::Config() decisionlib not specified; always caching files"); - return true; - } - else - { - libp = val; - } - - const char* params; - params = (val[0]) ? Config.GetWord() : 0; - - - XrdOucPinLoader* myLib = new XrdOucPinLoader(&m_log, 0, "decisionlib", - libp.c_str()); - - Decision *(*ep)(XrdSysError&); - ep = (Decision *(*)(XrdSysError&))myLib->Resolve("XrdFileCacheGetDecision"); - if (!ep) {myLib->Unload(true); return false;} - - Decision * d = ep(m_log); - if (!d) - { - clLog()->Error(XrdCl::AppMsg, "Factory::Config() decisionlib was not able to create a decision object"); - return false; - } - if (params) - d->ConfigDecision(params); - - m_decisionpoints.push_back(d); - clLog()->Info(XrdCl::AppMsg, "Factory::Config() successfully created decision lib from %s", libp.c_str()); - return true; -} -//______________________________________________________________________________ - - -bool Factory::Decide(XrdOucCacheIO* io) -{ - if(!m_decisionpoints.empty()) - { - std::string filename = io->Path(); - std::vector::const_iterator it; - for ( it = m_decisionpoints.begin(); it != m_decisionpoints.end(); ++it) - { - XrdFileCache::Decision *d = *it; - if (!d) continue; - if (!d->Decide(filename, *m_output_fs)) - { - return false; - } - } - } - - return true; -} - - - -//______________________________________________________________________________ - - -bool Factory::Config(XrdSysLogger *logger, const char *config_filename, const char *parameters) -{ - m_log.logger(logger); - - const char * cache_env; - if (!(cache_env = getenv("XRDPOSIX_CACHE")) || !*cache_env) - XrdOucEnv::Export("XRDPOSIX_CACHE", "mode=s&optwr=0"); - - XrdOucEnv myEnv; - XrdOucStream Config(&m_log, getenv("XRDINSTANCE"), &myEnv, "=====> "); - - if (!config_filename || !*config_filename) - { - clLog()->Warning(XrdCl::AppMsg, "Factory::Config() configuration file not specified."); - return false; - } - - int fd; - if ( (fd = open(config_filename, O_RDONLY, 0)) < 0) - { - clLog()->Error(XrdCl::AppMsg, "Factory::Config() can't open configuration file %s", config_filename); - return false; - } - - Config.Attach(fd); - - // Obtain plugin configurator - XrdOfsConfigPI *ofsCfg = XrdOfsConfigPI::New(config_filename,&Config,&m_log, - &XrdVERSIONINFOVAR(XrdOucGetCache)); - if (!ofsCfg) return false; - - - if (ofsCfg->Load(XrdOfsConfigPI::theOssLib)) { - ofsCfg->Plugin(m_output_fs); - XrdOssCache_FS* ocfs = XrdOssCache::Find("public"); - ocfs->Add(m_configuration.m_cache_dir.c_str()); - } - else - { - clLog()->Error(XrdCl::AppMsg, "Factory::Config() Unable to create an OSS object"); - m_output_fs = 0; - return false; - } - - - // Actual parsing of the config file. - bool retval = true; - char *var; - while((var = Config.GetMyFirstWord())) - { - if (!strcmp(var,"pfc.osslib")) - { - ofsCfg->Parse(XrdOfsConfigPI::theOssLib); - } - else if (!strcmp(var,"pfc.decisionlib")) - { - xdlib(Config); - } - else if (!strncmp(var,"pfc.", 4)) - { - retval = ConfigParameters(std::string(var+4), Config); - } - - if (!retval) - { - retval = false; - clLog()->Error(XrdCl::AppMsg, "Factory::Config() error in parsing"); - break; - } - - } - - Config.Close(); - // sets default value for disk usage - if (m_configuration.m_diskUsageLWM < 0 || m_configuration.m_diskUsageHWM < 0) - { - XrdOssVSInfo sP; - if (m_output_fs->StatVS(&sP, "public", 1) >= 0) { - m_configuration.m_diskUsageLWM = static_cast(0.90 * sP.Total + 0.5); - m_configuration.m_diskUsageHWM = static_cast(0.95 * sP.Total + 0.5); - clLog()->Debug(XrdCl::AppMsg, "Default disk usage [%lld, %lld]", m_configuration.m_diskUsageLWM, m_configuration.m_diskUsageHWM); - } - } - - if (retval) - { - int loff = 0; - char buff[2048]; - loff = snprintf(buff, sizeof(buff), "result\n" - "\tpfc.cachedir %s\n" - "\tpfc.blocksize %lld\n" - "\tpfc.nramread %d\n\tpfc.nramprefetch %d\n", - m_configuration.m_cache_dir.c_str() , - m_configuration.m_bufferSize, - m_configuration.m_NRamBuffersRead, m_configuration.m_NRamBuffersPrefetch ); - - if (m_configuration.m_hdfsmode) - { - char buff2[512]; - snprintf(buff2, sizeof(buff2), "\tpfc.hdfsmode hdfsbsize %lld\n", m_configuration.m_hdfsbsize); - loff += snprintf(&buff[loff], strlen(buff2), "%s", buff2); - } - - - char unameBuff[256]; - if (m_configuration.m_username.empty()) { - XrdOucUtils::UserName(getuid(), unameBuff, sizeof(unameBuff)); - m_configuration.m_username = unameBuff; - } - else { - snprintf(unameBuff, sizeof(unameBuff), "\tpfc.user %s \n", m_configuration.m_username.c_str()); - loff += snprintf(&buff[loff], strlen(unameBuff), "%s", unameBuff); - } - - m_log.Emsg("Config", buff); - } - - m_log.Emsg("Config", "Configuration = ", retval ? "Success" : "Fail"); - - if (ofsCfg) delete ofsCfg; - return retval; -} - -//______________________________________________________________________________ - - -bool Factory::ConfigParameters(std::string part, XrdOucStream& config ) -{ - XrdSysError err(0, ""); - if ( part == "user" ) - { - m_configuration.m_username = config.GetWord(); - } - else if ( part == "cachedir" ) - { - m_configuration.m_cache_dir = config.GetWord(); - } - else if ( part == "diskusage" ) - { - std::string minV = config.GetWord(); - std::string maxV = config.GetWord(); - if (!minV.empty() && !maxV.empty()) { - XrdOssVSInfo sP; - if (m_output_fs->StatVS(&sP, "public", 1) >= 0) - { - if (::isalpha(*(minV.rbegin())) && ::isalpha(*(minV.rbegin()))) { - if ( XrdOuca2x::a2sz(m_log, "Error getting disk usage low watermark", minV.c_str(), &m_configuration.m_diskUsageLWM, 0, sP.Total) - || XrdOuca2x::a2sz(m_log, "Error getting disk usage high watermark", maxV.c_str(), &m_configuration.m_diskUsageHWM, 0, sP.Total)) - { - return false; - } - } - else - { - char* eP; - errno = 0; - float lwmf = strtod(minV.c_str(), &eP); - if (errno || eP == minV.c_str()) { - m_log.Emsg("Factory::ConfigParameters() error parsing diskusage parameter ", minV.c_str()); - return false; - } - float hwmf = strtod(maxV.c_str(), &eP); - if (errno || eP == maxV.c_str()) { - m_log.Emsg("Factory::ConfigParameters() error parsing diskusage parameter ", maxV.c_str()); - return false; - } - - m_configuration.m_diskUsageLWM = static_cast(sP.Total * lwmf + 0.5); - m_configuration.m_diskUsageHWM = static_cast(sP.Total * hwmf + 0.5); - } - } - } - } - else if ( part == "blocksize" ) - { - long long minBSize = 64 * 1024; - long long maxBSize = 16 * 1024 * 1024; - if ( XrdOuca2x::a2sz(m_log, "get block size", config.GetWord(), &m_configuration.m_bufferSize, minBSize, maxBSize)) - { - return false; - } - } - else if (part == "nramread") - { - m_configuration.m_NRamBuffersRead = ::atoi(config.GetWord()); - } - else if (part == "nramprefetch") - { - m_configuration.m_NRamBuffersPrefetch = ::atoi(config.GetWord()); - } - else if ( part == "hdfsmode" ) - { - m_configuration.m_hdfsmode = true; - - const char* params = config.GetWord(); - if (params) { - if (!strncmp("hdfsbsize", params, 9)) { - long long minBlSize = 128 * 1024; - long long maxBlSize = 1024 * 1024 * 1024; - params = config.GetWord(); - if ( XrdOuca2x::a2sz(m_log, "Error getting file fragment size", params, &m_configuration.m_hdfsbsize, minBlSize, maxBlSize)) - { - return false; - } - } - else { - m_log.Emsg("Config", "Error setting the fragment size parameter name"); - return false; - } - } - } - else - { - m_log.Emsg("Factory::ConfigParameters() unmatched pfc parameter", part.c_str()); - return false; - } - - assert ( config.GetWord() == 0 && "Factory::ConfigParameters() lost argument"); - - return true; -} - -//______________________________________________________________________________ -//namespace { - -class FPurgeState -{ -public: - struct FS - { - std::string path; - long long nByte; - - FS(const char* p, long long n) : path(p), nByte(n) {} - }; - - typedef std::multimap map_t; - typedef map_t::iterator map_i; - - FPurgeState(long long iNByteReq) : nByteReq(iNByteReq), nByteAccum(0) {} - - map_t fmap; - - void checkFile (time_t iTime, const char* iPath, long long iNByte) - { - if (nByteAccum < nByteReq || iTime < fmap.rbegin()->first) - { - fmap.insert(std::pair (iTime, FS(iPath, iNByte))); - nByteAccum += iNByte; - - // remove newest files from map if necessary - while (nByteAccum > nByteReq) - { - time_t nt = fmap.begin()->first; - std::pair ret = fmap.equal_range(nt); - for (map_i it2 = ret.first; it2 != ret.second; ++it2) - nByteAccum -= it2->second.nByte; - fmap.erase(ret.first, ret.second); - } - } - } - -private: - long long nByteReq; - long long nByteAccum; -}; - - -//} - -void FillFileMapRecurse( XrdOssDF* iOssDF, const std::string& path, FPurgeState& purgeState) -{ - char buff[256]; - XrdOucEnv env; - int rdr; - const size_t InfoExtLen = strlen(XrdFileCache::Info::m_infoExtension); // cached var - XrdCl::Log *log = XrdCl::DefaultEnv::GetLog(); - - Factory& factory = Factory::GetInstance(); - while ((rdr = iOssDF->Readdir(&buff[0], 256)) >= 0) - { - // printf("readdir [%s]\n", buff); - std::string np = path + "/" + std::string(buff); - size_t fname_len = strlen(&buff[0]); - if (fname_len == 0) - { - // std::cout << "Finish read dir.[" << np <<"] Break loop \n"; - break; - } - - if (strncmp("..", &buff[0], 2) && strncmp(".", &buff[0], 1)) - { - XrdOssDF* dh = factory.GetOss()->newDir(factory.RefConfiguration().m_username.c_str()); - XrdOssDF* fh = factory.GetOss()->newFile(factory.RefConfiguration().m_username.c_str()); - - if (fname_len > InfoExtLen && strncmp(&buff[fname_len - InfoExtLen ], XrdFileCache::Info::m_infoExtension, InfoExtLen) == 0) - { - // XXXX MT - shouldn't we also check if it is currently opened? - - fh->Open(np.c_str(), O_RDONLY, 0600, env); - Info cinfo(factory.RefConfiguration().m_bufferSize); - time_t accessTime; - cinfo.Read(fh); - if (cinfo.GetLatestDetachTime(accessTime, fh)) - { - log->Debug(XrdCl::AppMsg, "FillFileMapRecurse() checking %s accessTime %d ", buff, (int)accessTime); - purgeState.checkFile(accessTime, np.c_str(), cinfo.GetNDownloadedBytes()); - } - else - { - // cinfo file does not contain any known accesses, use stat.mtime instead. - - log->Info(XrdCl::AppMsg, "FillFileMapRecurse() could not get access time for %s, trying stat.\n", np.c_str()); - - XrdOss* oss = Factory::GetInstance().GetOss(); - struct stat fstat; - - if (oss->Stat(np.c_str(), &fstat) == XrdOssOK) - { - accessTime = fstat.st_mtime; - log->Info(XrdCl::AppMsg, "FillFileMapRecurse() determined access time for %s via stat: %lld\n", - np.c_str(), accessTime); - - purgeState.checkFile(accessTime, np.c_str(), cinfo.GetNDownloadedBytes()); - } - else - { - // This really shouldn't happen ... but if it does remove cinfo and the data file right away. - - log->Warning(XrdCl::AppMsg, "FillFileMapRecurse() could not get access time for %s. Purging directly.\n", - np.c_str()); - - oss->Unlink(np.c_str()); - np = np.substr(0, np.size() - strlen(XrdFileCache::Info::m_infoExtension)); - oss->Unlink(np.c_str()); - } - } - } - else if (dh->Opendir(np.c_str(), env) >= 0) - { - FillFileMapRecurse(dh, np, purgeState); - } - - delete dh; dh = 0; - delete fh; fh = 0; - } - } -} - - -void Factory::CacheDirCleanup() -{ - // check state every sleep seconds - const static int sleept = 300; - struct stat fstat; - XrdOucEnv env; - - XrdOss* oss = Factory::GetInstance().GetOss(); - XrdOssVSInfo sP; - - while (1) - { - // get amount of space to erase - long long bytesToRemove = 0; - if (oss->StatVS(&sP, "public", 1) < 0) - { - clLog()->Error(XrdCl::AppMsg, "Factory::CacheDirCleanup() can't get statvs for dir [%s] \n", m_configuration.m_cache_dir.c_str()); - exit(1); - } - else - { - long long ausage = sP.Total - sP.Free; - clLog()->Info(XrdCl::AppMsg, "Factory::CacheDirCleanup() occupates disk space == %lld", ausage); - if (ausage > m_configuration.m_diskUsageHWM) - { - bytesToRemove = ausage - m_configuration.m_diskUsageLWM; - clLog()->Info(XrdCl::AppMsg, "Factory::CacheDirCleanup() need space for %lld bytes", bytesToRemove); - } - } - - if (bytesToRemove > 0) - { - // make a sorted map of file patch by access time - XrdOssDF* dh = oss->newDir(m_configuration.m_username.c_str()); - if (dh->Opendir(m_configuration.m_cache_dir.c_str(), env) >= 0) - { - FPurgeState purgeState(bytesToRemove * 5 / 4); // prepare 20% more volume than required - - FillFileMapRecurse(dh, m_configuration.m_cache_dir, purgeState); - - // loop over map and remove files with highest value of access time - for (FPurgeState::map_i it = purgeState.fmap.begin(); it != purgeState.fmap.end(); ++it) - { - // XXXX MT - shouldn't we re-check if the file is currently opened? - - std::string path = it->second.path; - // remove info file - if (oss->Stat(path.c_str(), &fstat) == XrdOssOK) - { - bytesToRemove -= fstat.st_size; - oss->Unlink(path.c_str()); - clLog()->Info(XrdCl::AppMsg, "Factory::CacheDirCleanup() removed %s size %lld", - path.c_str(), fstat.st_size); - } - - // remove data file - path = path.substr(0, path.size() - strlen(XrdFileCache::Info::m_infoExtension)); - if (oss->Stat(path.c_str(), &fstat) == XrdOssOK) - { - bytesToRemove -= it->second.nByte; - oss->Unlink(path.c_str()); - clLog()->Info(XrdCl::AppMsg, "Factory::CacheDirCleanup() removed %s bytes %lld, stat_size %lld", - path.c_str(), it->second.nByte, fstat.st_size); - } - - if (bytesToRemove <= 0) - break; - } - } - dh->Close(); - delete dh; dh =0; - } - - sleep(sleept); - } -} - diff --git a/src/XrdFileCache/XrdFileCacheFactory.hh b/src/XrdFileCache/XrdFileCacheFactory.hh deleted file mode 100644 index 5bc679cbbd4..00000000000 --- a/src/XrdFileCache/XrdFileCacheFactory.hh +++ /dev/null @@ -1,167 +0,0 @@ -#ifndef __XRDFILECACHE_FACTORY_HH__ -#define __XRDFILECACHE_FACTORY_HH__ -//---------------------------------------------------------------------------------- -// Copyright (c) 2014 by Board of Trustees of the Leland Stanford, Jr., University -// Author: Alja Mrak-Tadel, Matevz Tadel, Brian Bockelman -//---------------------------------------------------------------------------------- -// XRootD is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// XRootD is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with XRootD. If not, see . -//---------------------------------------------------------------------------------- - -#include -#include -#include - -#include "XrdSys/XrdSysPthread.hh" -#include "XrdOuc/XrdOucCache.hh" - -#include "XrdCl/XrdClDefaultEnv.hh" -#include "XrdVersion.hh" -#include "XrdFileCacheDecision.hh" - -class XrdOucStream; -class XrdSysError; - -namespace XrdCl -{ - class Log; -} - -namespace XrdFileCache -{ - //---------------------------------------------------------------------------- - //! Contains parameters configurable from the xrootd config file. - //---------------------------------------------------------------------------- - struct Configuration - { - Configuration() : - m_hdfsmode(false), - m_diskUsageLWM(-1), - m_diskUsageHWM(-1), - m_bufferSize(1024*1024), - m_NRamBuffersRead(8), - m_NRamBuffersPrefetch(1), - m_hdfsbsize(128*1024*1024) {} - - bool m_hdfsmode; //!< flag for enabling block-level operation - std::string m_cache_dir; //!< path of disk cache - std::string m_username; //!< username passed to oss plugin - - long long m_diskUsageLWM; //!< cache purge low water mark - long long m_diskUsageHWM; //!< cache purge high water mark - - long long m_bufferSize; //!< prefetch buffer size, default 1MB - int m_NRamBuffersRead; //!< number of read in-memory cache blocks - int m_NRamBuffersPrefetch; //!< number of prefetch in-memory cache blocks - long long m_hdfsbsize; //!< used with m_hdfsmode, default 128MB - }; - - - //---------------------------------------------------------------------------- - //! Instantiates Cache and Decision plugins. Parses configuration file. - //---------------------------------------------------------------------------- - class Factory : public XrdOucCache - { - public: - //-------------------------------------------------------------------------- - //! Constructor - //-------------------------------------------------------------------------- - Factory(); - - //--------------------------------------------------------------------- - //! \brief Unused abstract method. This method is implemented in the - //! the Cache class. - //--------------------------------------------------------------------- - virtual XrdOucCacheIO *Attach(XrdOucCacheIO *, int Options=0) { return NULL; } - - //--------------------------------------------------------------------- - //! \brief Unused abstract method. This information is available in - //! the Cache class. - //--------------------------------------------------------------------- - virtual int isAttached() { return false; } - - //--------------------------------------------------------------------- - //! Creates XrdFileCache::Cache object - //--------------------------------------------------------------------- - virtual XrdOucCache* Create(Parms &, XrdOucCacheIO::aprParms *aprP); - - XrdOss* GetOss() const { return m_output_fs; } - - //--------------------------------------------------------------------- - //! Getter for xrootd logger - //--------------------------------------------------------------------- - XrdSysError& GetSysError() { return m_log; } - - //-------------------------------------------------------------------- - //! \brief Makes decision if the original XrdOucCacheIO should be cached. - //! - //! @param & URL of file - //! - //! @return decision if IO object will be cached. - //-------------------------------------------------------------------- - bool Decide(XrdOucCacheIO*); - - //------------------------------------------------------------------------ - //! Reference XrdFileCache configuration - //------------------------------------------------------------------------ - const Configuration& RefConfiguration() const { return m_configuration; } - - - //--------------------------------------------------------------------- - //! \brief Parse configuration file - //! - //! @param logger xrootd logger - //! @param config_filename path to configuration file - //! @param parameters optional parameters to be passed - //! - //! @return parse status - //--------------------------------------------------------------------- - bool Config(XrdSysLogger *logger, const char *config_filename, const char *parameters); - - //--------------------------------------------------------------------- - //! Singleton access. - //--------------------------------------------------------------------- - static Factory &GetInstance(); - - //--------------------------------------------------------------------- - //! Version check. - //--------------------------------------------------------------------- - static bool VCheck(XrdVersionInfo &urVersion) { return true; } - - //--------------------------------------------------------------------- - //! Thread function running disk cache purge periodically. - //--------------------------------------------------------------------- - void CacheDirCleanup(); - - private: - bool ConfigParameters(std::string, XrdOucStream&); - bool ConfigXeq(char *, XrdOucStream &); - bool xdlib(XrdOucStream &); - - XrdCl::Log* clLog() const { return XrdCl::DefaultEnv::GetLog(); } - - static Factory *m_factory; //!< this object - - XrdSysError m_log; //!< XrdFileCache namespace logger - XrdOucCacheStats m_stats; //!< passed to cache, currently not used - XrdOss *m_output_fs; //!< disk cache file system - - std::vector m_decisionpoints; //!< decision plugins - - std::map m_filesInQueue; - - Configuration m_configuration; //!< configurable parameters - }; -} - -#endif diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc new file mode 100644 index 00000000000..41fc3170133 --- /dev/null +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -0,0 +1,1038 @@ +//---------------------------------------------------------------------------------- +// Copyright (c) 2014 by Board of Trustees of the Leland Stanford, Jr., University +// Author: Alja Mrak-Tadel, Matevz Tadel +//---------------------------------------------------------------------------------- +// XRootD is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// XRootD is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with XRootD. If not, see . +//---------------------------------------------------------------------------------- + + +#include "XrdFileCacheFile.hh" + + +#include +#include +#include +#include +#include "XrdCl/XrdClLog.hh" +#include "XrdCl/XrdClConstants.hh" +#include "XrdCl/XrdClFile.hh" +#include "XrdSys/XrdSysPthread.hh" +#include "XrdSys/XrdSysTimer.hh" +#include "XrdOss/XrdOss.hh" +#include "XrdOuc/XrdOucEnv.hh" +#include "Xrd/XrdScheduler.hh" +#include "XrdSfs/XrdSfsInterface.hh" +#include "XrdPosix/XrdPosixFile.hh" +#include "XrdPosix/XrdPosix.hh" +#include "XrdFileCacheFactory.hh" +#include "XrdFileCache.hh" +#include "Xrd/XrdScheduler.hh" + +using namespace XrdFileCache; + +namespace XrdPosixGlobals +{ + extern XrdScheduler *schedP; +} + +namespace +{ + const int PREFETCH_MAX_ATTEMPTS = 10; + + class DiskSyncer : public XrdJob + { + private: + File *m_file; + public: + DiskSyncer(File *pref, const char *desc="") : + XrdJob(desc), + m_file(pref) + {} + void DoIt() + { + m_file->Sync(); + } + }; +} + +namespace +{ + Cache* cache() { return Factory::GetInstance().GetCache(); } +} + +File::File(XrdOucCacheIO &inputIO, std::string& disk_file_path, long long iOffset, long long iFileSize) : +m_input(inputIO), +m_output(NULL), +m_infoFile(NULL), +m_cfi(Factory::GetInstance().RefConfiguration().m_bufferSize), +m_temp_filename(disk_file_path), +m_offset(iOffset), +m_fileSize(iFileSize), +m_stopping(false), +m_stateCond(0), // We will explicitly lock the condition before use. + +m_syncer(new DiskSyncer(this, "XrdFileCache::DiskSyncer")), +m_non_flushed_cnt(0), +m_in_sync(false), +m_downloadCond(0), +m_prefetchState(kOn), +m_prefetchReadCnt(0), +m_prefetchHitCnt(0), +m_prefetchScore(1), +m_prefetchCurrentCnt(0) +{ + clLog()->Debug(XrdCl::AppMsg, "File::File() %s", m_input.Path()); + if (!Open()) { + clLog()->Error(XrdCl::AppMsg, "File::File() Open failed %s !!!", m_input.Path()); + } +} + +void File::BlockRemovedFromWriteQ(Block* b) +{ + m_downloadCond.Lock(); + dec_ref_count(b); + clLog()->Dump(XrdCl::AppMsg, "File::BlockRemovedFromWriteQ() check write queues %p %d...%s", (void*)b, b->m_offset/m_cfi.GetBufferSize(), lPath()); + m_downloadCond.UnLock(); +} + +File::~File() +{ + clLog()->Debug(XrdCl::AppMsg, "File::~File() enter %p %s", (void*)this, lPath()); + + // assert if in debug mode + if (XrdCl::DefaultEnv::GetLog()->GetLevel() >= XrdCl::Log::DebugMsg ) { + m_stateCond.Lock(); + assert (m_stopping == true); + m_stateCond.UnLock(); + } + + cache()->RemoveWriteQEntriesFor(this); + + clLog()->Info(XrdCl::AppMsg, "File::~File() check write queues ...%s", lPath()); + + while (true) + { + m_stateCond.Lock(); + bool isPrefetching = (m_prefetchCurrentCnt > 0); + m_stateCond.UnLock(); + + if (isPrefetching == false) + { + m_downloadCond.Lock(); + // remove failed blocks + BlockMap_i itr = m_block_map.begin(); + while (itr != m_block_map.end()) { + if (itr->second->is_failed() && itr->second->m_refcnt == 1) { + BlockMap_i toErase = itr; + ++itr; + free_block(toErase->second); + } + else { + ++itr; + } + } + + bool blockMapEmpty = m_block_map.empty(); + int blocksize = (int)m_block_map.size(); + m_downloadCond.UnLock(); + if ( blockMapEmpty) + break; + clLog()->Info(XrdCl::AppMsg, "File::~File() mapsize %d %s", blocksize,lPath()); + for (BlockMap_i it = m_block_map.begin(); it != m_block_map.end(); ++it) { + Block* b = it->second; + clLog()->Debug(XrdCl::AppMsg, "File::~File() block idx=%d p=%d rcnt=%d dwnd=%d %s", + b->m_offset/m_cfi.GetBufferSize(), b->m_prefetch, b->m_refcnt, b->m_downloaded, lPath()); + } + } + XrdSysTimer::Wait(10); + } + clLog()->Debug(XrdCl::AppMsg, "File::~File finished with writing %s",lPath() ); + + + // Wait disk sync + bool do_sync = false; + { + XrdSysMutexHelper _lck(&m_syncStatusMutex); + if (m_non_flushed_cnt > 0 || !m_writes_during_sync.empty()) + { + do_sync = true; + m_in_sync = true; + clLog()->Info(XrdCl::AppMsg, "File::~File sync unflushed %d\n", m_non_flushed_cnt); + } + } + if (do_sync) + { + Sync(); + } + // write statistics in *cinfo file + AppendIOStatToFileInfo(); + + clLog()->Info(XrdCl::AppMsg, "File::~File close data file %p",(void*)this , lPath()); + if (m_output) + { + m_output->Close(); + delete m_output; + m_output = NULL; + } + if (m_infoFile) + { + clLog()->Info(XrdCl::AppMsg, "File::~File close info file"); + m_infoFile->Close(); + delete m_infoFile; + m_infoFile = NULL; + } + delete m_syncer; + + // print just for curiosity + clLog()->Debug(XrdCl::AppMsg, "File::~File() ended, prefetch score ...%d/%d=%.2f", m_prefetchHitCnt, m_prefetchReadCnt, m_prefetchScore); +} + +bool File::InitiateClose() +{ + // Retruns true if delay is needed + clLog()->Debug(XrdCl::AppMsg, "File::Initiate close start %s", lPath()); + + cache()->DeRegisterPrefetchFile(this); + + m_stateCond.Lock(); + m_stopping = true; + m_stateCond.UnLock(); + m_prefetchState = kCanceled; + if (m_cfi.IsComplete()) return false; // AMT maybe map size is here more meaningfull, but might hold block state lock + return true; +} + +//______________________________________________________________________________ + + + +//============================================================================== + +bool File::Open() +{ + clLog()->Dump(XrdCl::AppMsg, "File::Open() open file for disk cache %s", m_input.Path()); + + XrdOss &m_output_fs = *Factory::GetInstance().GetOss(); + // Create the data file itself. + XrdOucEnv myEnv; + m_output_fs.Create(Factory::GetInstance().RefConfiguration().m_username.c_str(), m_temp_filename.c_str(), 0600, myEnv, XRDOSS_mkpath); + m_output = m_output_fs.newFile(Factory::GetInstance().RefConfiguration().m_username.c_str()); + if (m_output) + { + int res = m_output->Open(m_temp_filename.c_str(), O_RDWR, 0600, myEnv); + if (res < 0) + { + clLog()->Error(XrdCl::AppMsg, "File::Open() can't get data-FD for %s %s", m_temp_filename.c_str(), m_input.Path()); + delete m_output; + m_output = 0; + + return false; + } + } + else + { + clLog()->Error(XrdCl::AppMsg, "File::Open() can't get data holder "); + return false; + } + + // Create the info file + std::string ifn = m_temp_filename + Info::m_infoExtension; + m_output_fs.Create(Factory::GetInstance().RefConfiguration().m_username.c_str(), ifn.c_str(), 0600, myEnv, XRDOSS_mkpath); + m_infoFile = m_output_fs.newFile(Factory::GetInstance().RefConfiguration().m_username.c_str()); + if (m_infoFile) + { + int res = m_infoFile->Open(ifn.c_str(), O_RDWR, 0600, myEnv); + if (res < 0) + { + clLog()->Error(XrdCl::AppMsg, "File::Open() can't get info-FD %s %s", ifn.c_str(), m_input.Path()); + delete m_infoFile; + m_infoFile = 0; + return false; + } + } + else + { + return false; + } + + if (m_cfi.Read(m_infoFile, Factory::GetInstance().RefConfiguration().m_prefetch) <= 0) + { + int ss = (m_fileSize - 1)/m_cfi.GetBufferSize() + 1; + clLog()->Info(XrdCl::AppMsg, "Creating new file info with size %lld. Reserve space for %d blocks %s", m_fileSize, ss, m_input.Path()); + m_cfi.ResizeBits(ss, Factory::GetInstance().RefConfiguration().m_prefetch); + m_cfi.WriteHeader(m_infoFile); + } + else + { + clLog()->Debug(XrdCl::AppMsg, "Info file read from disk: %s", m_input.Path()); + } + + + cache()->RegisterPrefetchFile(this); + return true; +} + + +//============================================================================== +// Read and helpers +//============================================================================== + + + +//namespace +//{ +bool File::overlap(int blk, // block to query + long long blk_size, // + long long req_off, // offset of user request + int req_size, // size of user request + // output: + long long &off, // offset in user buffer + long long &blk_off, // offset in block + long long &size) // size to copy + { + const long long beg = blk * blk_size; + const long long end = beg + blk_size; + const long long req_end = req_off + req_size; + + if (req_off < end && req_end > beg) + { + const long long ovlp_beg = std::max(beg, req_off); + const long long ovlp_end = std::min(end, req_end); + + off = ovlp_beg - req_off; + blk_off = ovlp_beg - beg; + size = ovlp_end - ovlp_beg; + + assert(size <= blk_size); + return true; + } + else + { + return false; + } + } +//} + +//------------------------------------------------------------------------------ + +Block* File::RequestBlock(int i, bool prefetch) +{ + // Must be called w/ block_map locked. + // Checks on size etc should be done before. + // + // Reference count is 0 so increase it in calling function if you want to + // catch the block while still in memory. + clLog()->Debug(XrdCl::AppMsg, "RequestBlock() %d pOn=(%d)", i, prefetch); + + XrdCl::File &client = ((XrdPosixFile*)(&m_input))->clFile; + + const long long BS = m_cfi.GetBufferSize(); + const int last_block = m_cfi.GetSizeInBits() - 1; + + long long off = i * BS; + long long this_bs = (i == last_block) ? m_input.FSize() - off : BS; + + Block *b = new Block(this, off, this_bs, prefetch); // should block be reused to avoid recreation + + XrdCl::XRootDStatus status = client.Read(off, this_bs, (void*)b->get_buff(), new BlockResponseHandler(b)); + if (status.IsOK()) { + clLog()->Dump(XrdCl::AppMsg, "File::RequestBlock() this = %p, b=%p, this idx=%d pOn=(%d) %s", (void*)this, (void*)b, i, prefetch, lPath()); + m_block_map[i] = b; + + if (m_prefetchState == kOn && m_block_map.size() > Factory::GetInstance().RefConfiguration().m_prefetch_max_blocks) + { + m_prefetchState = kHold; + cache()->DeRegisterPrefetchFile(this); + } + return b; + } + else { + clLog()->Error(XrdCl::AppMsg, "File::RequestBlock() error %d, this = %p, b=%p, this idx=%d pOn=(%d) %s", status.code, (void*)this, (void*)b, i, prefetch, lPath()); + XrdPosixMap::Result(status); + return 0; + } +} + +//------------------------------------------------------------------------------ + +int File::RequestBlocksDirect(DirectResponseHandler *handler, IntList_t& blocks, + char* req_buf, long long req_off, long long req_size) +{ + XrdCl::File &client = ((XrdPosixFile*)(&m_input))->clFile; + + const long long BS = m_cfi.GetBufferSize(); + + // XXX Use readv to load more at the same time. + + long long total = 0; + + for (IntList_i ii = blocks.begin() ; ii != blocks.end(); ++ii) + { + // overlap and request + long long off; // offset in user buffer + long long blk_off; // offset in block + long long size; // size to copy + + overlap(*ii, BS, req_off, req_size, off, blk_off, size); + + XrdCl::Status status = client.Read(*ii * BS + blk_off, size, req_buf + off, handler); + if (!status.IsOK()) + { + clLog()->Error(XrdCl::AppMsg, "File::RequestBlocksDirect error %s\n", lPath()); + XrdPosixMap::Result(status); + return -1; // AMT all reads should be canceled in this case + } + else { + clLog()->Dump(XrdCl::AppMsg, "RequestBlockDirect success %d %ld %s", *ii, size, lPath()); + } + total += size; + } + + return total; +} + +//------------------------------------------------------------------------------ + +int File::ReadBlocksFromDisk(std::list& blocks, + char* req_buf, long long req_off, long long req_size) +{ + + clLog()->Dump(XrdCl::AppMsg, "File::ReadBlocksFromDisk %ld %s", blocks.size(), lPath()); + const long long BS = m_cfi.GetBufferSize(); + + long long total = 0; + + // XXX Coalesce adjacent reads. + + for (IntList_i ii = blocks.begin() ; ii != blocks.end(); ++ii) + { + // overlap and read + long long off; // offset in user buffer + long long blk_off; // offset in block + long long size; // size to copy + + overlap(*ii, BS, req_off, req_size, off, blk_off, size); + + long long rs = m_output->Read(req_buf + off, *ii * BS + blk_off, size); + clLog()->Dump(XrdCl::AppMsg, "File::ReadBlocksFromDisk block %d size %d %s", *ii, size, lPath()); + + + if (rs < 0) { + clLog()->Error(XrdCl::AppMsg, "File::ReadBlocksFromDisk neg retval %ld (%ld@%d) %s", rs, *ii * BS + blk_off, lPath()); + return rs; + } + + + // AMT I think we should exit in this case too + if (rs !=size) { + clLog()->Error(XrdCl::AppMsg, "File::ReadBlocksFromDisk incomplete %ld (%ld@%d) %s", rs, *ii * BS + blk_off, lPath()); + return -1; + } + + total += rs; + + + CheckPrefetchStatDisk(*ii); + } + + return total; +} + +//------------------------------------------------------------------------------ + +int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) +{ + const long long BS = m_cfi.GetBufferSize(); + + // lock + // loop over reqired blocks: + // - if on disk, ok; + // - if in ram or incoming, inc ref-count + // - if not available, request and inc ref count + // before requesting the hell and more (esp. for sparse readv) assess if + // passing the req to client is actually better. + // unlock + + bool preProcOK = true; + m_downloadCond.Lock(); + + // XXX Check for blocks to free? Later ... + + const int idx_first = iUserOff / BS; + const int idx_last = (iUserOff + iUserSize - 1) / BS; + + BlockList_t blks_to_process, blks_processed; + IntList_t blks_on_disk, blks_direct; + + for (int block_idx = idx_first; block_idx <= idx_last; ++block_idx) + { + clLog()->Dump(XrdCl::AppMsg, "--- File::Read() idx %d %s \n", block_idx, lPath()); + BlockMap_i bi = m_block_map.find(block_idx); + + // In RAM or incoming? + if (bi != m_block_map.end()) + { + // XXXX if failed before -- retry if timestamp sufficient or fail? + // XXXX Or just push it and handle errors in one place later? + + inc_ref_count(bi->second); + clLog()->Dump(XrdCl::AppMsg, "File::Read() u=%p inc_ref_count for existing block %p %d %s", (void*)iUserBuff, (void*)bi->second, block_idx, lPath()); + blks_to_process.push_front(bi->second); + m_stats.m_BytesRam++; // AMT what if block fails + } + // On disk? + else if (m_cfi.TestBit(block_idx)) + { + clLog()->Dump(XrdCl::AppMsg, "File::Read() u=%p read from disk %d %s", (void*)iUserBuff, block_idx, lPath()); + blks_on_disk.push_back(block_idx); + m_stats.m_BytesDisk++; + } + // Then we have to get it ... + else + { + // Is there room for one more RAM Block? + if ( cache()->HaveFreeWritingSlots() && cache()->RequestRAMBlock()) + { + clLog()->Dump(XrdCl::AppMsg, "File::Read() u=%p inc_ref_count new %d %s", (void*)iUserBuff, block_idx, lPath()); + Block *b = RequestBlock(block_idx, false); + // assert(b); + if (!b) { + preProcOK = false; + break; + } + inc_ref_count(b); + blks_to_process.push_back(b); + m_stats.m_BytesRam++; + } + // Nope ... read this directly without caching. + else + { + clLog()->Debug(XrdCl::AppMsg, "File::Read() direct block %d %s", block_idx, lPath()); + blks_direct.push_back(block_idx); + m_stats.m_BytesMissed++; + } + } + + } + + m_downloadCond.UnLock(); + + if (!preProcOK) { + for (BlockList_i i = blks_to_process.begin(); i!= blks_to_process.end(); ++i ) + dec_ref_count(*i); + return -1; // AMT ??? + } + + long long bytes_read = 0; + + // First, send out any direct requests. + // XXX Could send them all out in a single vector read. + DirectResponseHandler *direct_handler = 0; + int direct_size = 0; + + if (!blks_direct.empty()) + { + direct_handler = new DirectResponseHandler(blks_direct.size()); + + direct_size = RequestBlocksDirect(direct_handler, blks_direct, iUserBuff, iUserOff, iUserSize); + // failed to send direct client request + if (direct_size < 0) { + for (BlockList_i i = blks_to_process.begin(); i!= blks_to_process.end(); ++i ) + dec_ref_count(*i); + delete direct_handler; + return -1; // AMT ??? + } + clLog()->Dump(XrdCl::AppMsg, "File::Read() direct read %d. %s", direct_size, lPath()); + } + + // Second, read blocks from disk. + if ((!blks_on_disk.empty()) && (bytes_read >= 0)) { + int rc = ReadBlocksFromDisk(blks_on_disk, iUserBuff, iUserOff, iUserSize); + clLog()->Dump(XrdCl::AppMsg, "File::Read() u=%p, from disk %d. %s", (void*)iUserBuff, rc, lPath()); + if (rc >= 0) + { + bytes_read += rc; + } + else + { + bytes_read = rc; + clLog()->Error(XrdCl::AppMsg, "File::Read() failed to read from disk. %s", lPath()); + // AMT commented line below should not be an immediate return, can have block refcount increased and map increased + // return rc; + } + } + + // Third, loop over blocks that are available or incoming + while ( (! blks_to_process.empty()) && (bytes_read >= 0)) + { + BlockList_t finished; + + { + XrdSysCondVarHelper _lck(m_downloadCond); + + BlockList_i bi = blks_to_process.begin(); + while (bi != blks_to_process.end()) + { + // clLog()->Dump(XrdCl::AppMsg, "File::Read() searcing for block %p finished", (void*)(*bi)); + if ((*bi)->is_finished()) + { + clLog()->Dump(XrdCl::AppMsg, "File::Read() found finished block %p %s", (void*)(*bi), lPath()); + finished.push_back(*bi); + BlockList_i bj = bi++; + blks_to_process.erase(bj); + } + else + { + ++bi; + } + } + + if (finished.empty()) + { + + clLog()->Dump(XrdCl::AppMsg, "File::Read() wait block begin %s", lPath()); + + m_downloadCond.Wait(); + + clLog()->Dump(XrdCl::AppMsg, "File::Read() wait block end %s", lPath()); + + continue; + } + } + + clLog()->Dump(XrdCl::AppMsg, "File::Read() bytes read before processing blocks %d %s\n", bytes_read, lPath()); + + BlockList_i bi = finished.begin(); + while (bi != finished.end()) + { + if ((*bi)->is_ok()) + { + long long user_off; // offset in user buffer + long long off_in_block; // offset in block + long long size_to_copy; // size to copy + + // clLog()->Dump(XrdCl::AppMsg, "File::Read() Block finished ok."); + overlap((*bi)->m_offset/BS, BS, iUserOff, iUserSize, user_off, off_in_block, size_to_copy); + + clLog()->Dump(XrdCl::AppMsg, "File::Read() u=%p, from finished block %d , size %d end %s", (void*)iUserBuff, (*bi)->m_offset/BS, size_to_copy, lPath()); + memcpy(&iUserBuff[user_off], &((*bi)->m_buff[off_in_block]), size_to_copy); + bytes_read += size_to_copy; + + CheckPrefetchStatRAM(*bi); + } + else // it has failed ... krap up. + { + clLog()->Error(XrdCl::AppMsg, "File::Read() Block finished with error %s.", lPath()); + bytes_read = -1; + errno = (*bi)->m_errno; + break; + } + ++bi; + } + + std::copy(finished.begin(), finished.end(), std::back_inserter(blks_processed)); + finished.clear(); + } + + clLog()->Dump(XrdCl::AppMsg, "File::Read() bytes read after processing blocks %d %s\n", bytes_read, lPath()); + + // Fourth, make sure all direct requests have arrived + if ((direct_handler != 0) && (bytes_read >= 0 )) + { + clLog()->Debug(XrdCl::AppMsg, "File::Read() waiting for direct requests %s.", lPath()); + XrdSysCondVarHelper _lck(direct_handler->m_cond); + + if (direct_handler->m_to_wait > 0) + { + direct_handler->m_cond.Wait(); + } + + if (direct_handler->m_errno == 0) + { + bytes_read += direct_size; + } + else + { + errno = direct_handler->m_errno; + bytes_read = -1; + } + + delete direct_handler; + } + clLog()->Debug(XrdCl::AppMsg, "File::Read() before assert %s.", lPath()); + assert(iUserSize >= bytes_read); + + // Last, stamp and release blocks, release file. + { + XrdSysCondVarHelper _lck(m_downloadCond); + + // AMT what is stamp block ??? + + // blks_to_process can be non-empty, if we're exiting with an error. + std::copy(blks_to_process.begin(), blks_to_process.end(), std::back_inserter(blks_processed)); + + for (BlockList_i bi = blks_processed.begin(); bi != blks_processed.end(); ++bi) + { + clLog()->Dump(XrdCl::AppMsg, "File::Read() dec_ref_count b=%p, %d %s", (void*)(*bi), ((*bi)->m_offset/BufferSize()), lPath()); + dec_ref_count(*bi); + // XXXX stamp block + } + } + + return bytes_read; +} + +//------------------------------------------------------------------------------ + +void File::WriteBlockToDisk(Block* b) +{ + int retval = 0; + // write block buffer into disk file + long long offset = b->m_offset - m_offset; + long long size = (b->m_offset + m_cfi.GetBufferSize()) > m_input.FSize() ? (m_input.FSize() - b->m_offset) : m_cfi.GetBufferSize(); + int buffer_remaining = size; + int buffer_offset = 0; + int cnt = 0; + const char* buff = &b->m_buff[0]; + while ((buffer_remaining > 0) && // There is more to be written + (((retval = m_output->Write(buff, offset + buffer_offset, buffer_remaining)) != -1) + || (errno == EINTR))) // Write occurs without an error + { + buffer_remaining -= retval; + buff += retval; + cnt++; + + if (buffer_remaining) + { + clLog()->Warning(XrdCl::AppMsg, "File::WriteToDisk() reattempt[%d] writing missing %ld for block %d %s", + cnt, buffer_remaining, b->m_offset, lPath()); + } + if (cnt > PREFETCH_MAX_ATTEMPTS) + { + clLog()->Error(XrdCl::AppMsg, "File::WriteToDisk() write failed too manny attempts %s", lPath()); + return; + } + } + + // set bit fetched + clLog()->Dump(XrdCl::AppMsg, "File::WriteToDisk() success set bit for block [%ld] size [%d] %s", b->m_offset, size, lPath()); + int pfIdx = (b->m_offset - m_offset)/m_cfi.GetBufferSize(); + + m_downloadCond.Lock(); + assert((m_cfi.TestBit(pfIdx) == false) && "Block not yet fetched."); + m_cfi.SetBitFetched(pfIdx); + m_downloadCond.UnLock(); + + { + XrdSysCondVarHelper _lck(m_downloadCond); + // clLog()->Dump(XrdCl::AppMsg, "File::WriteToDisk() dec_ref_count %d %s", pfIdx, lPath()); + dec_ref_count(b); + } + + // set bit synced + bool schedule_sync = false; + { + XrdSysMutexHelper _lck(&m_syncStatusMutex); + + if (m_in_sync) + { + m_writes_during_sync.push_back(pfIdx); + } + else + { + m_cfi.SetBitWriteCalled(pfIdx); + ++m_non_flushed_cnt; + } + + if (m_non_flushed_cnt >= 100 && (m_cfi.IsComplete() && m_non_flushed_cnt > 0)) + { + schedule_sync = true; + m_in_sync = true; + m_non_flushed_cnt = 0; + } + } + + if (schedule_sync) + { + XrdPosixGlobals::schedP->Schedule(m_syncer); + } +} + +//------------------------------------------------------------------------------ + +void File::Sync() +{ + clLog()->Dump(XrdCl::AppMsg, "File::Sync %s", lPath()); + m_output->Fsync(); + m_cfi.WriteHeader(m_infoFile); + int written_while_in_sync; + { + XrdSysMutexHelper _lck(&m_syncStatusMutex); + for (std::vector::iterator i = m_writes_during_sync.begin(); i != m_writes_during_sync.end(); ++i) + { + m_cfi.SetBitWriteCalled(*i); + } + written_while_in_sync = m_non_flushed_cnt = (int) m_writes_during_sync.size(); + m_writes_during_sync.clear(); + m_in_sync = false; + } + clLog()->Dump(XrdCl::AppMsg, "File::Sync %d blocks written during sync.", written_while_in_sync); + m_infoFile->Fsync(); +} + +//______________________________________________________________________________ + +void File::inc_ref_count(Block* b) +{ + // Method always called under lock + b->m_refcnt++; + clLog()->Dump(XrdCl::AppMsg, "File::inc_ref_count b=%p, %d %s ",(void*)b, b->m_refcnt, lPath()); +} + +//______________________________________________________________________________ + +void File::dec_ref_count(Block* b) +{ + // Method always called under lock + b-> m_refcnt--; + assert(b->m_refcnt >= 0); + + //AMT ... this is ugly, ... File::Read() can decrease ref count before waiting to be , prefetch starts with refcnt 0 + if ( b->m_refcnt == 0 && b->is_finished()) { + free_block(b); + } +} + +void File::free_block(Block* b) +{ + int i = b->m_offset/BufferSize(); + clLog()->Dump(XrdCl::AppMsg, "File::free_block block (%p) %d %s ", (void*)b, i, lPath()); + delete m_block_map[i]; + size_t ret = m_block_map.erase(i); + if (ret != 1) + { + clLog()->Error(XrdCl::AppMsg, "File::OnBlockZeroRefCount did not erase %d from map.", i); + } + else + { + cache()->RAMBlockReleased(); + } + + if (m_prefetchState == kHold && m_block_map.size() < Factory::GetInstance().RefConfiguration().m_prefetch_max_blocks) + { + m_prefetchState = kOn; + cache()->RegisterPrefetchFile(this); + } +} + +//------------------------------------------------------------------------------ + +void File::ProcessBlockResponse(Block* b, XrdCl::XRootDStatus *status) +{ + + m_downloadCond.Lock(); + + clLog()->Debug(XrdCl::AppMsg, "File::ProcessBlockResponse %p, %d %s",(void*)b,(int)(b->m_offset/BufferSize()), lPath()); + if (status->IsOK()) + { + b->m_downloaded = true; + clLog()->Debug(XrdCl::AppMsg, "File::ProcessBlockResponse %d finished %d %s",(int)(b->m_offset/BufferSize()), b->is_finished(), lPath()); + if (!m_stopping) { // AMT theoretically this should be under state lock, but then are double locks + clLog()->Debug(XrdCl::AppMsg, "File::ProcessBlockResponse inc_ref_count %d %s\n", (int)(b->m_offset/BufferSize()), lPath()); + inc_ref_count(b); + cache()->AddWriteTask(b, true); + } + else { + // there is no refcount +/- to remove dropped prefetched blocks on destruction + if (b->m_prefetch && (b->m_refcnt == 0)) + free_block(b); + } + } + else + { + // AMT how long to keep? + // when to retry? + clLog()->Error(XrdCl::AppMsg, "File::ProcessBlockResponse block %p %d error=%d, [%s] %s",(void*)b,(int)(b->m_offset/BufferSize()), status->code, status->GetErrorMessage().c_str(), lPath()); + XrdPosixMap::Result(*status); + // AMT could notfiy global cache we dont need RAM for that block + b->set_error_and_free(errno); + errno = 0; + + // ??? AMT how long to keep + inc_ref_count(b); + } + + m_downloadCond.Broadcast(); + + m_downloadCond.UnLock(); +} + + + + long long File::BufferSize() { + return m_cfi.GetBufferSize(); + } + +//______________________________________________________________________________ +const char* File::lPath() const +{ +return m_temp_filename.c_str(); +} + +//______________________________________________________________________________ +void File::AppendIOStatToFileInfo() +{ + // lock in case several IOs want to write in *cinfo file + if (m_infoFile) + { + Info::AStat as; + as.DetachTime = time(0); + as.BytesDisk = m_stats.m_BytesDisk; + as.BytesRam = m_stats.m_BytesRam; + as.BytesMissed = m_stats.m_BytesMissed; + m_cfi.AppendIOStat(as, (XrdOssDF*)m_infoFile); + } + else + { + clLog()->Warning(XrdCl::AppMsg, "File::AppendIOStatToFileInfo() info file not opened %s", lPath()); + } +} + +//______________________________________________________________________________ +void File::Prefetch() +{ + if (m_prefetchState == kOn) + { + // clLog()->Dump(XrdCl::AppMsg, "File::Prefetch enter to check download status \n"); + XrdSysCondVarHelper _lck(m_downloadCond); + // clLog()->Dump(XrdCl::AppMsg, "File::Prefetch enter to check download status BEGIN %s \n", lPath()); + + // check index not on disk and not in RAM + bool found = false; + for (int f=0; f < m_cfi.GetSizeInBits(); ++f) + { + // clLog()->Dump(XrdCl::AppMsg, "File::Prefetch test bit %d", f); + if (!m_cfi.TestBit(f)) + { + BlockMap_i bi = m_block_map.find(f); + if (bi == m_block_map.end()) { + clLog()->Dump(XrdCl::AppMsg, "File::Prefetch take block %d %s", f, lPath()); + cache()->RequestRAMBlock(); + RequestBlock(f, true); + m_prefetchReadCnt++; + m_prefetchScore = float(m_prefetchHitCnt)/m_prefetchReadCnt; + found = true; + break; + } + } + } + if (!found) { + clLog()->Dump(XrdCl::AppMsg, "File::Prefetch no free blcok found "); + m_cfi.CheckComplete(); + // assert (m_cfi.IsComplete()); + // it is possible all missing blocks are in map but downlaoded status is still not complete + clLog()->Dump(XrdCl::AppMsg, "File::Prefetch -- unlikely to happen ... file seem to be complete %s", lPath()); + // remove block from map + cache()->DeRegisterPrefetchFile(this); + } + clLog()->Dump(XrdCl::AppMsg, "File::Prefetch end"); + } + + UnMarkPrefetch(); +} + + +//______________________________________________________________________________ +void File::CheckPrefetchStatRAM(Block* b) +{ + if (Factory::GetInstance().RefConfiguration().m_prefetch) { + if (b->m_prefetch) { + m_prefetchHitCnt++; + m_prefetchScore = float(m_prefetchHitCnt)/m_prefetchReadCnt; + } + } +} + +//______________________________________________________________________________ +void File::CheckPrefetchStatDisk(int idx) +{ + if (Factory::GetInstance().RefConfiguration().m_prefetch) { + if (m_cfi.TestPrefetchBit(idx)) + m_prefetchHitCnt++; + } +} + +//______________________________________________________________________________ +float File::GetPrefetchScore() const +{ + return m_prefetchScore; +} + +//______________________________________________________________________________ +void File::MarkPrefetch() +{ + m_stateCond.Lock(); + m_prefetchCurrentCnt++; + m_stateCond.UnLock(); + +} + +//______________________________________________________________________________ +void File::UnMarkPrefetch() +{ + m_stateCond.Lock(); + m_prefetchCurrentCnt--; + m_stateCond.UnLock(); +} + +//============================================================================== +//================== RESPONSE HANDLER ================================== +//============================================================================== + +void BlockResponseHandler::HandleResponse(XrdCl::XRootDStatus *status, + XrdCl::AnyObject *response) +{ + XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg,"BlockResponseHandler::HandleResponse()"); + + m_block->m_file->ProcessBlockResponse(m_block, status); + + delete status; + delete response; + + delete this; +} + +//------------------------------------------------------------------------------ + +void DirectResponseHandler::HandleResponse(XrdCl::XRootDStatus *status, + XrdCl::AnyObject *response) +{ + XrdCl::DefaultEnv::GetLog()->Dump(XrdCl::AppMsg,"DirectResponseHandler::HandleRespons()"); + XrdSysCondVarHelper _lck(m_cond); + + --m_to_wait; + + if ( ! status->IsOK()) + { + XrdPosixMap::Result(*status); + m_errno = errno; + } + + if (m_to_wait == 0) + { + m_cond.Signal(); + } +} + + diff --git a/src/XrdFileCache/XrdFileCacheFile.hh b/src/XrdFileCache/XrdFileCacheFile.hh new file mode 100644 index 00000000000..1a98651cc45 --- /dev/null +++ b/src/XrdFileCache/XrdFileCacheFile.hh @@ -0,0 +1,263 @@ +#ifndef __XRDFILECACHE_FILE_HH__ +#define __XRDFILECACHE_FILE_HH__ +//---------------------------------------------------------------------------------- +// Copyright (c) 2014 by Board of Trustees of the Leland Stanford, Jr., University +// Author: Alja Mrak-Tadel, Matevz Tadel +//---------------------------------------------------------------------------------- +// XRootD is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// XRootD is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with XRootD. If not, see . +//---------------------------------------------------------------------------------- + +#include "XrdCl/XrdClXRootDResponses.hh" +#include "XrdCl/XrdClDefaultEnv.hh" + +#include "XrdFileCacheInfo.hh" +#include "XrdFileCacheStats.hh" + +#include +#include + +class XrdJob; +class XrdOucIOVec; + +namespace XrdCl +{ + class Log; +} + +namespace XrdFileCache +{ + class BlockResponseHandler; + class DirectResponseHandler; + + struct ReadVBlockListRAM; + struct ReadVChunkListRAM; + struct ReadVBlockListDisk; + struct ReadVChunkListDisk; +} + + +namespace XrdFileCache +{ + class RefCounted + { + int m_refcnt; + + RefCounted() : m_refcnt(0) {} + }; + + class File; + + class Block + { + public: + std::vector m_buff; + long long m_offset; + File *m_file; + bool m_prefetch; + int m_refcnt; + int m_errno; + bool m_downloaded; + + Block(File *f, long long off, int size, bool m_prefetch) : + m_offset(off), m_file(f), m_prefetch(m_prefetch), m_refcnt(0), + m_errno(0), m_downloaded(false) + { + m_buff.resize(size); + } + + const char* get_buff(long long pos = 0) const { return &m_buff[pos]; } + + bool is_finished() { return m_downloaded || m_errno != 0; } + bool is_ok() { return m_downloaded; } + bool is_failed() { return m_errno != 0; } + + void set_error_and_free(int err) + { + m_errno = err; + m_buff.resize(0); + } + }; + + class File + { + private: + enum PrefetchState_e { kOn, kHold, kCanceled }; + + XrdOucCacheIO &m_input; //!< original data source + XrdOssDF *m_output; //!< file handle for data file on disk + XrdOssDF *m_infoFile; //!< file handle for data-info file on disk + Info m_cfi; //!< download status of file blocks and access statistics + + std::string m_temp_filename; //!< filename of data file on disk + long long m_offset; //!< offset of cached file for block-based operation + long long m_fileSize; //!< size of cached disk file for block-based operation + + bool m_stopping; //!< run thread should be stopped + + XrdSysCondVar m_stateCond; //!< state condition variable + + // fsync + XrdSysMutex m_syncStatusMutex; //!< mutex locking fsync status + XrdJob *m_syncer; + std::vector m_writes_during_sync; + int m_non_flushed_cnt; + bool m_in_sync; + + typedef std::list IntList_t; + typedef IntList_t::iterator IntList_i; + + typedef std::list BlockList_t; + typedef BlockList_t::iterator BlockList_i; + + typedef std::map BlockMap_t; + typedef BlockMap_t::iterator BlockMap_i; + + + BlockMap_t m_block_map; + + XrdSysCondVar m_downloadCond; + + Stats m_stats; //!< cache statistics, used in IO detach + + PrefetchState_e m_prefetchState; + + int m_prefetchReadCnt; + int m_prefetchHitCnt; + float m_prefetchScore; //cached + int m_prefetchCurrentCnt; + + public: + //------------------------------------------------------------------------ + //! Constructor. + //------------------------------------------------------------------------ + File(XrdOucCacheIO &io, std::string &path, + long long offset, long long fileSize); + + //------------------------------------------------------------------------ + //! Destructor. + //------------------------------------------------------------------------ + ~File(); + + void BlockRemovedFromWriteQ(Block*); + //! Open file handle for data file and info file on local disk. + bool Open(); + + //! Vector read from disk if block is already downloaded, else ReadV from client. + int ReadV (const XrdOucIOVec *readV, int n); + + int Read(char* buff, long long offset, int size); + + //---------------------------------------------------------------------- + //! \brief Initiate close. Return true if still IO active. + //! Used in XrdPosixXrootd::Close() + //---------------------------------------------------------------------- + bool InitiateClose(); + + //---------------------------------------------------------------------- + //! Sync file cache inf o and output data with disk + //---------------------------------------------------------------------- + void Sync(); + + //---------------------------------------------------------------------- + //! Reference to prefetch statistics. + //---------------------------------------------------------------------- + Stats& GetStats() { return m_stats; } + + void ProcessBlockResponse(Block* b, XrdCl::XRootDStatus *status); + void WriteBlockToDisk(Block* b); + + void Prefetch(); + + float GetPrefetchScore() const; + + void MarkPrefetch(); + + //! Log path + const char* lPath() const; + + private: + bool overlap(int blk, // block to query + long long blk_size, // + long long req_off, // offset of user request + int req_size, // size of user request + // output: + long long &off, // offset in user buffer + long long &blk_off, // offset in block + long long &size); + // Read + Block* RequestBlock(int i, bool prefetch); + + int RequestBlocksDirect(DirectResponseHandler *handler, IntList_t& blocks, + char* buff, long long req_off, long long req_size); + + int ReadBlocksFromDisk(IntList_t& blocks, + char* req_buf, long long req_off, long long req_size); + + // VRead + bool VReadPreProcess(const XrdOucIOVec *readV, int n, ReadVBlockListRAM& blks_to_process, ReadVBlockListDisk& blks_on_disk, XrdCl::ChunkList& chunkVec); + int VReadFromDisk(const XrdOucIOVec *readV, int n, ReadVBlockListDisk& blks_on_disk); + int VReadProcessBlocks(const XrdOucIOVec *readV, int n, std::vector& blks_to_process, std::vector& blks_rocessed); + + + long long BufferSize(); + void AppendIOStatToFileInfo(); + + void CheckPrefetchStatRAM(Block* b); + void CheckPrefetchStatDisk(int idx); + + void UnMarkPrefetch(); + + //! Short log alias. + XrdCl::Log* clLog() const { return XrdCl::DefaultEnv::GetLog(); } + + void inc_ref_count(Block*); + void dec_ref_count(Block*); + void free_block(Block*); + + }; + + + // ================================================================ + + class BlockResponseHandler : public XrdCl::ResponseHandler + { + public: + Block *m_block; + + BlockResponseHandler(Block *b) : m_block(b) {} + + void HandleResponse(XrdCl::XRootDStatus *status, + XrdCl::AnyObject *response); + }; + + class DirectResponseHandler : public XrdCl::ResponseHandler + { + public: + XrdSysCondVar m_cond; + int m_to_wait; + int m_errno; + + DirectResponseHandler(int to_wait) : m_cond(0), m_to_wait(to_wait), m_errno(0) {} + + bool is_finished() { XrdSysCondVarHelper _lck(m_cond); return m_to_wait == 0; } + bool is_ok() { XrdSysCondVarHelper _lck(m_cond); return m_to_wait == 0 && m_errno == 0; } + bool is_failed() { XrdSysCondVarHelper _lck(m_cond); return m_errno != 0; } + + void HandleResponse(XrdCl::XRootDStatus *status, + XrdCl::AnyObject *response); + }; + +} + +#endif diff --git a/src/XrdFileCache/XrdFileCacheIO.hh b/src/XrdFileCache/XrdFileCacheIO.hh new file mode 100644 index 00000000000..df2b0fd18c8 --- /dev/null +++ b/src/XrdFileCache/XrdFileCacheIO.hh @@ -0,0 +1,45 @@ +#ifndef __XRDFILECACHE_CACHE_IO_HH__ +#define __XRDFILECACHE_CACHE_IO_HH__ + +#include "XrdFileCache.hh" +#include "XrdOuc/XrdOucCache.hh" +#include "XrdCl/XrdClDefaultEnv.hh" + +namespace XrdFileCache +{ + //---------------------------------------------------------------------------- + //! Base cache-io class that implements XrdOucCacheIO abstract methods. + //---------------------------------------------------------------------------- + class IO : public XrdOucCacheIO + { + public: + IO (XrdOucCacheIO &io, XrdOucCacheStats &stats, Cache &cache) : + m_io(io), m_statsGlobal(stats), m_cache(cache) {} + + //! Original data source. + virtual XrdOucCacheIO *Base() { return &m_io; } + + //! Original data source URL. + virtual long long FSize() { return m_io.FSize(); } + + //! Original data source URL. + virtual const char *Path() { return m_io.Path(); } + + virtual int Sync() { return 0; } + + virtual int Trunc(long long Offset) { errno = ENOTSUP; return -1; } + + virtual int Write(char *Buffer, long long Offset, int Length) + { errno = ENOTSUP; return -1; } + + + protected: + XrdCl::Log* clLog() const { return XrdCl::DefaultEnv::GetLog(); } + + XrdOucCacheIO &m_io; //!< original data source + XrdOucCacheStats &m_statsGlobal; //!< reference to Cache statistics + Cache &m_cache; //!< reference to Cache needed in detach + }; +} + +#endif diff --git a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc index ed009b61ec9..e7728db67e2 100644 --- a/src/XrdFileCache/XrdFileCacheIOEntireFile.cc +++ b/src/XrdFileCache/XrdFileCacheIOEntireFile.cc @@ -29,27 +29,19 @@ using namespace XrdFileCache; -void *PrefetchRunner(void * prefetch_void) -{ - XrdFileCache::Prefetch *prefetch = static_cast(prefetch_void); - if (prefetch) - prefetch->Run(); - return NULL; -} //______________________________________________________________________________ IOEntireFile::IOEntireFile(XrdOucCacheIO &io, XrdOucCacheStats &stats, Cache & cache) : IO(io, stats, cache), - m_prefetch(0) + m_file(0) { clLog()->Info(XrdCl::AppMsg, "IO::IO() [%p] %s", this, m_io.Path()); + + XrdCl::URL url(io.Path()); + std::string fname = Factory::GetInstance().RefConfiguration().m_cache_dir + url.GetPath(); - std::string fname; - m_cache.getFilePathFromURL(io.Path(), fname); - - m_prefetch = new Prefetch(io, fname, 0, io.FSize()); - + m_file = new File(io, fname, 0, io.FSize()); } IOEntireFile::~IOEntireFile() @@ -57,24 +49,17 @@ IOEntireFile::~IOEntireFile() bool IOEntireFile::ioActive() { - return m_prefetch->InitiateClose(); -} - -void IOEntireFile::StartPrefetch() -{ - pthread_t tid; - XrdSysThread::Run(&tid, PrefetchRunner, (void *)(m_prefetch), 0, "XrdFileCache Prefetcher"); + return m_file->InitiateClose(); } - XrdOucCacheIO *IOEntireFile::Detach() { - m_statsGlobal.Add(m_prefetch->GetStats()); + m_statsGlobal.Add(m_file->GetStats()); XrdOucCacheIO * io = &m_io; - delete m_prefetch; - m_prefetch = 0; + delete m_file; + m_file = 0; // This will delete us! m_cache.Detach(this); @@ -83,7 +68,7 @@ XrdOucCacheIO *IOEntireFile::Detach() int IOEntireFile::Read (char *buff, long long off, int size) { - clLog()->Debug(XrdCl::AppMsg, "IO::Read() [%p] %lld@%d %s", this, off, size, m_io.Path()); + clLog()->Debug(XrdCl::AppMsg, "IOEntireFile::Read() [%p] %lld@%d %s", this, off, size, m_io.Path()); // protect from reads over the file size if (off >= m_io.FSize()) @@ -99,34 +84,33 @@ int IOEntireFile::Read (char *buff, long long off, int size) ssize_t bytes_read = 0; ssize_t retval = 0; - retval = m_prefetch->Read(buff, off, size); - clLog()->Debug(XrdCl::AppMsg, "IO::Read() read from prefetch retval = %d %s", retval, m_io.Path()); - if (retval > 0) + retval = m_file->Read(buff, off, size); + clLog()->Debug(XrdCl::AppMsg, "IOEntireFile::Read() read from File retval = %d %s", retval, m_io.Path()); + if (retval >= 0) { bytes_read += retval; buff += retval; size -= retval; - if ((size > 0)) - clLog()->Debug(XrdCl::AppMsg, "IO::Read() missed %d bytes %s", size, m_io.Path()); - } - if (retval < 0) + if (size > 0) + clLog()->Warning(XrdCl::AppMsg, "IOEntireFile::Read() missed %d bytes %s", size, m_io.Path()); + } + else { - clLog()->Error(XrdCl::AppMsg, "IO::Read(), origin bytes read %d %s", retval, m_io.Path()); + clLog()->Error(XrdCl::AppMsg, "IOEntireFile::Read(), origin bytes read %d %s", retval, m_io.Path()); } return (retval < 0) ? retval : bytes_read; } - /* * Perform a readv from the cache */ int IOEntireFile::ReadV (const XrdOucIOVec *readV, int n) { - clLog()->Warning(XrdCl::AppMsg, "IO::ReadV(), get %d requests %s", n, m_io.Path()); + clLog()->Warning(XrdCl::AppMsg, "IOEntireFile::ReadV(), get %d requests %s", n, m_io.Path()); - return m_prefetch->ReadV(readV, n); + return m_file->ReadV(readV, n); } diff --git a/src/XrdFileCache/XrdFileCacheIOEntireFile.hh b/src/XrdFileCache/XrdFileCacheIOEntireFile.hh index 9917f174d8c..c9019bd4db2 100644 --- a/src/XrdFileCache/XrdFileCacheIOEntireFile.hh +++ b/src/XrdFileCache/XrdFileCacheIOEntireFile.hh @@ -21,9 +21,10 @@ #include #include "XrdSys/XrdSysPthread.hh" +#include "XrdFileCacheIO.hh" #include "XrdFileCache.hh" #include "XrdFileCacheStats.hh" -#include "XrdFileCachePrefetch.hh" +#include "XrdFileCacheFile.hh" class XrdSysError; class XrdOssDF; @@ -50,7 +51,7 @@ namespace XrdFileCache ~IOEntireFile(); //--------------------------------------------------------------------- - //! Pass Read request to the corresponding Prefetch object. + //! Pass Read request to the corresponding File object. //! //! @param Buffer //! @param Offset @@ -61,7 +62,7 @@ namespace XrdFileCache virtual int Read(char *Buffer, long long Offset, int Length); //--------------------------------------------------------------------- - //! Pass ReadV request to the corresponding Prefetch object. + //! Pass ReadV request to the corresponding File object. //! //! @param readV //! @param n number of XrdOucIOVecs @@ -81,12 +82,8 @@ namespace XrdFileCache //! Called to check if destruction needs to be done in a separate task. virtual bool ioActive(); - protected: - //! Run prefetch outside constructor. - virtual void StartPrefetch(); - private: - Prefetch* m_prefetch; + File* m_file; }; } diff --git a/src/XrdFileCache/XrdFileCacheIOFileBlock.cc b/src/XrdFileCache/XrdFileCacheIOFileBlock.cc index e2ef7ae55e4..1b63bf17c11 100644 --- a/src/XrdFileCache/XrdFileCacheIOFileBlock.cc +++ b/src/XrdFileCache/XrdFileCacheIOFileBlock.cc @@ -33,13 +33,6 @@ using namespace XrdFileCache; -void *PrefetchRunnerBl(void * prefetch_void) -{ - Prefetch *prefetch = static_cast(prefetch_void); - prefetch->Run(); - return NULL; -} - //______________________________________________________________________________ IOFileBlock::IOFileBlock(XrdOucCacheIO &io, XrdOucCacheStats &statsGlobal, Cache & cache) : IO(io, statsGlobal, cache) @@ -55,7 +48,7 @@ XrdOucCacheIO* IOFileBlock::Detach() XrdOucCacheIO * io = &m_io; - for (std::map::iterator it = m_blocks.begin(); it != m_blocks.end(); ++it) + for (std::map::iterator it = m_blocks.begin(); it != m_blocks.end(); ++it) { m_statsGlobal.Add(it->second->GetStats()); delete it->second; @@ -91,10 +84,11 @@ void IOFileBlock::GetBlockSizeFromPath() } //______________________________________________________________________________ -Prefetch* IOFileBlock::newBlockPrefetcher(long long off, int blocksize, XrdOucCacheIO* io) +File* IOFileBlock::newBlockFile(long long off, int blocksize, XrdOucCacheIO* io) { - std::string fname; - m_cache.getFilePathFromURL(io->Path(), fname); + XrdCl::URL url(io->Path()); + std::string fname = Factory::GetInstance().RefConfiguration().m_cache_dir + url.GetPath(); + std::stringstream ss; ss << fname; char offExt[64]; @@ -103,10 +97,8 @@ Prefetch* IOFileBlock::newBlockPrefetcher(long long off, int blocksize, XrdOucCa ss << &offExt[0]; fname = ss.str(); - clLog()->Debug(XrdCl::AppMsg, "FileBlock::FileBlock(), create XrdFileCachePrefetch. %s", m_io.Path()); - Prefetch* prefetch = new Prefetch(*io, fname, off, blocksize); - pthread_t tid; - XrdSysThread::Run(&tid, PrefetchRunnerBl, (void *)prefetch, 0, "BlockFile Prefetcher"); + clLog()->Debug(XrdCl::AppMsg, "FileBlock::FileBlock(), create XrdFileCacheFile. %s", m_io.Path()); + File* prefetch = new File(*io, fname, off, blocksize); return prefetch; } @@ -115,7 +107,7 @@ Prefetch* IOFileBlock::newBlockPrefetcher(long long off, int blocksize, XrdOucCa bool IOFileBlock::ioActive() { bool res = false; - for (std::map::iterator it = m_blocks.begin(); it != m_blocks.end(); ++it) { + for (std::map::iterator it = m_blocks.begin(); it != m_blocks.end(); ++it) { if (it->second->InitiateClose()) res = true; } @@ -146,9 +138,9 @@ int IOFileBlock::Read (char *buff, long long off, int size) for (int blockIdx = idx_first; blockIdx <= idx_last; ++blockIdx ) { // locate block - Prefetch* fb; + File* fb; m_mutex.Lock(); - std::map::iterator it = m_blocks.find(blockIdx); + std::map::iterator it = m_blocks.find(blockIdx); if ( it != m_blocks.end() ) { fb = it->second; @@ -164,8 +156,8 @@ int IOFileBlock::Read (char *buff, long long off, int size) clLog()->Debug(XrdCl::AppMsg, "IOFileBlock::Read() last block, change output file size to %lld \n %s", pbs, m_io.Path()); } - fb = newBlockPrefetcher(blockIdx*m_blocksize, pbs, &m_io); - m_blocks.insert(std::pair(blockIdx, (Prefetch*) fb)); + fb = newBlockFile(blockIdx*m_blocksize, pbs, &m_io); + m_blocks.insert(std::pair(blockIdx, (File*) fb)); } m_mutex.UnLock(); @@ -217,5 +209,3 @@ int IOFileBlock::Read (char *buff, long long off, int size) return bytes_read; } - - diff --git a/src/XrdFileCache/XrdFileCacheIOFileBlock.hh b/src/XrdFileCache/XrdFileCacheIOFileBlock.hh index 1d0af73e6f6..b11cf139ab6 100644 --- a/src/XrdFileCache/XrdFileCacheIOFileBlock.hh +++ b/src/XrdFileCache/XrdFileCacheIOFileBlock.hh @@ -23,8 +23,7 @@ #include "XrdOuc/XrdOucCache.hh" #include "XrdSys/XrdSysPthread.hh" -#include "XrdFileCache.hh" -#include "XrdFileCachePrefetch.hh" +#include "XrdFileCacheIO.hh" class XrdSysError; class XrdOssDF; @@ -57,7 +56,7 @@ namespace XrdFileCache virtual XrdOucCacheIO *Detach(); //--------------------------------------------------------------------- - //! Pass Read request to the corresponding Prefetch object. + //! Pass Read request to the corresponding File object. //--------------------------------------------------------------------- virtual int Read(char *Buffer, long long Offset, int Length); @@ -67,11 +66,11 @@ namespace XrdFileCache private: long long m_blocksize; //!< size of file-block - std::map m_blocks; //!< map of created blocks + std::map m_blocks; //!< map of created blocks XrdSysMutex m_mutex; //!< map mutex void GetBlockSizeFromPath(); - Prefetch* newBlockPrefetcher(long long off, int blocksize, XrdOucCacheIO* io); + File* newBlockFile(long long off, int blocksize, XrdOucCacheIO* io); }; } diff --git a/src/XrdFileCache/XrdFileCacheInfo.cc b/src/XrdFileCache/XrdFileCacheInfo.cc index 131c6161440..b08d8cbc51b 100644 --- a/src/XrdFileCache/XrdFileCacheInfo.cc +++ b/src/XrdFileCache/XrdFileCacheInfo.cc @@ -39,7 +39,8 @@ using namespace XrdFileCache; Info::Info(long long iBufferSize) : m_version(0), m_bufferSize(iBufferSize), - m_sizeInBits(0), m_buff_fetched(0), m_buff_write_called(0), + m_sizeInBits(0), + m_buff_fetched(0), m_buff_write_called(0), m_buff_prefetch(0), m_accessCnt(0), m_complete(false) { @@ -49,27 +50,32 @@ Info::~Info() { if (m_buff_fetched) free(m_buff_fetched); if (m_buff_write_called) free(m_buff_write_called); + if (m_buff_prefetch) free(m_buff_prefetch); } //______________________________________________________________________________ -void Info::ResizeBits(int s) +void Info::ResizeBits(int s, bool init_prefetch_buff) { m_sizeInBits = s; m_buff_fetched = (unsigned char*)malloc(GetSizeInBytes()); m_buff_write_called = (unsigned char*)malloc(GetSizeInBytes()); memset(m_buff_fetched, 0, GetSizeInBytes()); memset(m_buff_write_called, 0, GetSizeInBytes()); + if (init_prefetch_buff) { + m_buff_prefetch = (unsigned char*)malloc(GetSizeInBytes()); + memset(m_buff_prefetch, 0, GetSizeInBytes()); + } } //______________________________________________________________________________ -int Info::Read(XrdOssDF* fp) +int Info::Read(XrdOssDF* fp, bool init_prefetch_buff ) { - // does not need lock, called only in Prefetch::Open - // before Prefetch::Run() starts + // does not need lock, called only in File::Open + // before File::Run() starts int off = 0; off += fp->Read(&m_version, off, sizeof(int)); @@ -89,6 +95,13 @@ int Info::Read(XrdOssDF* fp) off += fp->Read(&m_accessCnt, off, sizeof(int)); clLog()->Dump(XrdCl::AppMsg, "Info:::Read() complete %d access_cnt %d", m_complete, m_accessCnt); + + + if (init_prefetch_buff) { + m_buff_prefetch = (unsigned char*)malloc(GetSizeInBytes()); + memset(m_buff_prefetch, 0, GetSizeInBytes()); + } + return off; } diff --git a/src/XrdFileCache/XrdFileCacheInfo.hh b/src/XrdFileCache/XrdFileCacheInfo.hh index 13f81f4e8de..e4378710d94 100644 --- a/src/XrdFileCache/XrdFileCacheInfo.hh +++ b/src/XrdFileCache/XrdFileCacheInfo.hh @@ -79,12 +79,19 @@ namespace XrdFileCache //--------------------------------------------------------------------- void SetBitWriteCalled(int i); + //! \brief Mark block as written from prefetch + //! + //! @param i block index + //--------------------------------------------------------------------- + void SetBitPrefetch(int i); + + //--------------------------------------------------------------------- //! \brief Reserve buffer for fileSize/bufferSize bytes //! //! @param n number of file blocks //--------------------------------------------------------------------- - void ResizeBits(int n); + void ResizeBits(int n, bool prefetch_stat = false); //--------------------------------------------------------------------- //! \brief Rea load content from cinfo file into this object @@ -93,12 +100,12 @@ namespace XrdFileCache //! //! @return number of bytes read //--------------------------------------------------------------------- - int Read(XrdOssDF* fp); + int Read(XrdOssDF* fp, bool init_prefetch = false); //--------------------------------------------------------------------- - //! Write number of blocks and prefetch buffer size + //! Write number of blocks and read buffer size //--------------------------------------------------------------------- - void WriteHeader(XrdOssDF* fp); + void WriteHeader(XrdOssDF* fp); //--------------------------------------------------------------------- //! Append access time, and cache statistics @@ -140,6 +147,11 @@ namespace XrdFileCache //--------------------------------------------------------------------- bool TestBit(int i) const; + //--------------------------------------------------------------------- + //! Test if block at the given index is prefetched + //--------------------------------------------------------------------- + bool TestPrefetchBit(int i) const; + //--------------------------------------------------------------------- //! Get complete status //--------------------------------------------------------------------- @@ -186,6 +198,7 @@ namespace XrdFileCache int m_sizeInBits; //!< number of file blocks unsigned char *m_buff_fetched; //!< download state vector unsigned char *m_buff_write_called; //!< disk written state vector + unsigned char *m_buff_prefetch; //!< prefetch state vector int m_accessCnt; //!< number of written AStat structs bool m_complete; //!< cached }; @@ -199,6 +212,15 @@ namespace XrdFileCache return (m_buff_fetched[cn] & cfiBIT(off)) == cfiBIT(off); } + // AMT could have only one function to test bit and pass an argument, but would loose clarity + inline bool Info::TestPrefetchBit(int i) const + { + int cn = i/8; + assert(cn < GetSizeInBytes()); + + int off = i - cn*8; + return (m_buff_prefetch[cn] & cfiBIT(off)) == cfiBIT(off); + } inline int Info::GetNDownloadedBlocks() const { @@ -260,6 +282,16 @@ namespace XrdFileCache m_buff_fetched[cn] |= cfiBIT(off); } + inline void Info::SetBitPrefetch(int i) + { + int cn = i/8; + assert(cn < GetSizeInBytes()); + + int off = i - cn*8; + m_buff_prefetch[cn] |= cfiBIT(off); + } + + inline long long Info::GetBufferSize() const { return m_bufferSize; diff --git a/src/XrdFileCache/XrdFileCachePrefetch.cc b/src/XrdFileCache/XrdFileCachePrefetch.cc deleted file mode 100644 index 9d4bed9ae35..00000000000 --- a/src/XrdFileCache/XrdFileCachePrefetch.cc +++ /dev/null @@ -1,1018 +0,0 @@ -//---------------------------------------------------------------------------------- -// Copyright (c) 2014 by Board of Trustees of the Leland Stanford, Jr., University -// Author: Alja Mrak-Tadel, Matevz Tadel, Brian Bockelman -//---------------------------------------------------------------------------------- -// XRootD is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// XRootD is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with XRootD. If not, see . -//---------------------------------------------------------------------------------- - -#include -#include -#include - -#include "XrdCl/XrdClLog.hh" -#include "XrdCl/XrdClConstants.hh" -#include "XrdCl/XrdClFile.hh" -#include "XrdSys/XrdSysPthread.hh" -#include "XrdSys/XrdSysTimer.hh" -#include "XrdOss/XrdOss.hh" -#include "XrdOuc/XrdOucEnv.hh" -#include "Xrd/XrdScheduler.hh" - -#include "XrdSfs/XrdSfsInterface.hh" -#include "XrdPosix/XrdPosixFile.hh" - -#include "XrdFileCachePrefetch.hh" -#include "XrdFileCacheFactory.hh" -#include "XrdFileCache.hh" - -using namespace XrdFileCache; - -namespace XrdPosixGlobals -{ - extern XrdScheduler *schedP; -} - -namespace -{ - const int PREFETCH_MAX_ATTEMPTS = 10; - - class DiskSyncer : public XrdJob - { - private: - Prefetch *m_prefetch; - - public: - DiskSyncer(Prefetch *pref, const char *desc="") : - XrdJob(desc), - m_prefetch(pref) - {} - - void DoIt() - { - m_prefetch->Sync(); - } - }; -} - - -Prefetch::RAM::RAM():m_numBlocks(0),m_buffer(0), m_blockStates(0), m_writeMutex(0) -{ - m_numBlocks = Factory::GetInstance().RefConfiguration().m_NRamBuffersRead + Factory::GetInstance().RefConfiguration().m_NRamBuffersPrefetch; - m_buffer = (char*)malloc(m_numBlocks * Factory::GetInstance().RefConfiguration().m_bufferSize); - m_blockStates = new RAMBlock[m_numBlocks]; -} - -Prefetch::RAM::~RAM() -{ - free(m_buffer); - delete [] m_blockStates; -} - -Prefetch::Prefetch(XrdOucCacheIO &inputIO, std::string& disk_file_path, long long iOffset, long long iFileSize) : - m_output(NULL), - m_infoFile(NULL), - m_cfi(Factory::GetInstance().RefConfiguration().m_bufferSize), - m_input(inputIO), - m_temp_filename(disk_file_path), - m_offset(iOffset), - m_fileSize(iFileSize), - m_started(false), - m_failed(false), - m_stopping(false), - m_stopped(false), - m_stateCond(0), // We will explicitly lock the condition before use. - m_queueCond(0), - m_syncer(new DiskSyncer(this, "XrdFileCache::DiskSyncer")), - m_non_flushed_cnt(0), - m_in_sync(false) -{ - assert(m_fileSize > 0); - clLog()->Debug(XrdCl::AppMsg, "Prefetch::Prefetch() %p %s", (void*)&m_input, lPath()); -} -//______________________________________________________________________________ - -bool Prefetch::InitiateClose() -{ - // Retruns true if delay is needed - - clLog()->Debug(XrdCl::AppMsg, "Prefetch::Initiate close start", lPath()); - if (m_cfi.IsComplete()) return false; - - XrdSysCondVarHelper monitor(m_stateCond); - m_stopping = true; - if (m_started == false) - { - m_stopped = true; - return false; - } - - return true; -} - -//______________________________________________________________________________ -Prefetch::~Prefetch() -{ - clLog()->Debug(XrdCl::AppMsg, "Prefetch::~Prefetch() %p %s", (void*)this, lPath()); - - m_queueCond.Lock(); - m_queueCond.Signal(); - m_queueCond.UnLock(); - - Cache::RemoveWriteQEntriesFor(this); - clLog()->Info(XrdCl::AppMsg, "Prefetch::~Prefetch() check write queues ...%s", lPath()); - while (true) - { - m_stateCond.Lock(); - bool isStopped = m_stopped; - m_stateCond.UnLock(); - - if (isStopped) - { - clLog()->Debug(XrdCl::AppMsg, "Prefetch::~Prefetch sleep, waiting queues to empty begin %s", lPath()); - - bool writewait = false; - m_ram.m_writeMutex.Lock(); - for (int i = 0; i < m_ram.m_numBlocks;++i ) - { - if (m_ram.m_blockStates[i].refCount) - { - writewait = true; - break; - } - } - m_ram.m_writeMutex.UnLock(); - - // disk sync - { - XrdSysMutexHelper _lck(&m_syncStatusMutex); - - if (m_in_sync) writewait = true; - } - - if (!writewait) - break; - } - - XrdSysTimer::Wait(100); - } - clLog()->Debug(XrdCl::AppMsg, "Prefetch::~Prefetch finished with writing %s",lPath() ); - - bool do_sync = false; - { - XrdSysMutexHelper _lck(&m_syncStatusMutex); - if (m_non_flushed_cnt > 0) - { - do_sync = true; - m_in_sync = true; - - clLog()->Info(XrdCl::AppMsg, "Prefetch::~Prefetch sync unflushed %d\n", m_non_flushed_cnt); - } - } - if (do_sync) - { - Sync(); - } - - if (m_output) - { - clLog()->Info(XrdCl::AppMsg, "Prefetch::~Prefetch close data file %p",(void*)this , lPath()); - - m_output->Close(); - delete m_output; - m_output = NULL; - } - else - { - clLog()->Info(XrdCl::AppMsg, "Prefetch::~Prefetch close data file -- not opened %p",(void*)this , lPath()); - } - if (m_infoFile) - { - clLog()->Info(XrdCl::AppMsg, "Prefetch::~Prefetch close info file"); - - // write statistics in *cinfo file - AppendIOStatToFileInfo(); - - m_infoFile->Close(); - delete m_infoFile; - m_infoFile = NULL; - } - else - { - clLog()->Info(XrdCl::AppMsg, "Prefetch::~Prefetch close info file -- not opened %p",(void*)this , lPath()); - } - - delete m_syncer; -} - -//______________________________________________________________________________ -const char* Prefetch::lPath() const -{ - return m_temp_filename.c_str(); -} - -//______________________________________________________________________________ - -bool Prefetch::Open() -{ - // clLog()->Debug(XrdCl::AppMsg, "Prefetch::Open() open file for disk cache %s", lPath()); - XrdOss &output_fs = *Factory::GetInstance().GetOss(); - // Create the data file itself. - XrdOucEnv myEnv; - output_fs.Create(Factory::GetInstance().RefConfiguration().m_username.c_str(), m_temp_filename.c_str(), 0644, myEnv, XRDOSS_mkpath); - m_output = output_fs.newFile(Factory::GetInstance().RefConfiguration().m_username.c_str()); - if (m_output) - { - int res = m_output->Open(m_temp_filename.c_str(), O_RDWR, 0644, myEnv); - if ( res < 0) - { - clLog()->Error(XrdCl::AppMsg, "Prefetch::Open() can't open local file %s", m_temp_filename.c_str()); - delete m_output; - m_output = NULL; - return false; - } - } - else - { - clLog()->Error(XrdCl::AppMsg, "Prefetch::Open() can't get data holder "); - return false; - } - - // Create the info file - std::string ifn = m_temp_filename + Info::m_infoExtension; - output_fs.Create(Factory::GetInstance().RefConfiguration().m_username.c_str(), ifn.c_str(), 0644, myEnv, XRDOSS_mkpath); - m_infoFile = output_fs.newFile(Factory::GetInstance().RefConfiguration().m_username.c_str()); - if (m_infoFile) - { - - int res = m_infoFile->Open(ifn.c_str(), O_RDWR, 0644, myEnv); - if ( res < 0 ) - { - clLog()->Error(XrdCl::AppMsg, "Prefetch::Open() can't get info-FD %s %s", ifn.c_str(), lPath()); - delete m_output; - m_output = NULL; - delete m_infoFile; - m_infoFile = NULL; - - return false; - } - } - if (!m_infoFile) - { - return false; - } - if ( m_cfi.Read(m_infoFile) <= 0) - { - assert(m_fileSize > 0); - int ss = (m_fileSize -1)/m_cfi.GetBufferSize() + 1; - // clLog()->Info(XrdCl::AppMsg, "Creating new file info with size %lld. Reserve space for %d blocks %s", m_fileSize, ss, lPath()); - m_cfi.ResizeBits(ss); - m_cfi.WriteHeader(m_infoFile); - } - else - { - clLog()->Debug(XrdCl::AppMsg, "Info file already exists %s", lPath()); - // m_cfi.Print(); - } - - return true; -} - - -//_________________________________________________________________________________________________ -void -Prefetch::Run() -{ - { - XrdSysCondVarHelper monitor(m_stateCond); - - if (m_started) - { - clLog()->Error(XrdCl::AppMsg, "Prefetch::Run() Already started for %s", lPath()); - m_stopped = true; - return; - } - - if (m_stopped) - { - return; - } - - if ( !Open()) - { - m_failed = true; - } - m_started = true; - // Broadcast to possible io-read waiting objects - m_stateCond.Broadcast(); - - if (m_failed) - { - m_stopped = true; - return; - } - } - assert(m_infoFile); - clLog()->Debug(XrdCl::AppMsg, "Prefetch::Run() Starting loop over tasks for %s", lPath()); - - Task* task; - int numReadBlocks = 0; - while ((task = GetNextTask()) != 0) - { - DoTask(task); - - if (task->condVar) - { - clLog()->Debug(XrdCl::AppMsg, "Prefetch::Run() task %p condvar %p", task, task->condVar); - XrdSysCondVarHelper tmph(task->condVar); - task->condVar->Signal(); - } - - clLog()->Debug(XrdCl::AppMsg, "Prefetch::Run() delete task %p condvar %p", task, task->condVar); - delete task; - - numReadBlocks++; - } // loop tasks - - - clLog()->Debug(XrdCl::AppMsg, "Prefetch::Run() exits, download %s !", m_cfi.IsComplete() ? " completed " : "unfinished %s", lPath()); - - - m_cfi.CheckComplete(); - - m_stateCond.Lock(); - m_stopped = true; - m_stateCond.UnLock(); -} // end Run() - - -//______________________________________________________________________________ -Prefetch::Task* -Prefetch::CreateTaskForFirstUndownloadedBlock() -{ - // first check if there are enough write and ram resources - if (Cache::HaveFreeWritingSlots() == false) return 0; - - int nRP = 0; - for (int i =0 ; i < m_ram.m_numBlocks; ++i) { - if (m_ram.m_blockStates[i].fromRead == false && m_ram.m_blockStates[i].refCount > 0) nRP++; - } - if ( nRP >= Factory::GetInstance().RefConfiguration().m_NRamBuffersPrefetch ) { - clLog()->Dump(XrdCl::AppMsg, "Prefetch::CreateTaskForFirstUndownloadedBlock no resources %d %d, %s ", nRP, Factory::GetInstance().RefConfiguration().m_NRamBuffersPrefetch, lPath()); - return 0; - } - - Task *task = new Task; - Task &t = * task; - t.ramBlockIdx = -1; - int fileBlockIdx = -1; - for (int f = 0; f < m_cfi.GetSizeInBits(); ++f) - { - m_downloadStatusMutex.Lock(); - bool isdn = m_cfi.TestBit(f); - m_downloadStatusMutex.UnLock(); - - if (!isdn) - { - fileBlockIdx = f + m_offset/m_cfi.GetBufferSize(); - // get ram for the file block - m_ram.m_writeMutex.Lock(); - for (int r =0 ; r < m_ram.m_numBlocks; ++r) - { - - if (m_ram.m_blockStates[r].fileBlockIdx == fileBlockIdx) - { - // skip this task, the file block f is already downloaded - break; - } - - if (m_ram.m_blockStates[r].refCount == 0 ) - { - t.ramBlockIdx = r; - - assert(m_ram.m_blockStates[r].fileBlockIdx == -1); - m_ram.m_blockStates[r].refCount = 1; - m_ram.m_blockStates[r].fileBlockIdx = fileBlockIdx; - m_ram.m_blockStates[r].status = kReadWait; - break; - } - } - m_ram.m_writeMutex.UnLock(); - - break; - } - } - - if (t.ramBlockIdx >= 0) { - clLog()->Dump(XrdCl::AppMsg, "Prefetch::CreateTaskForFirstUndownloadedBlock success block %d %s ", fileBlockIdx, lPath()); - return task; - } - else if (fileBlockIdx == -1) { - m_cfi.CheckComplete(); - } - - delete task; - return 0; -} - -//_____________________________________________________________________________ -Prefetch::Task* -Prefetch::GetNextTask() -{ - while (true) - { - m_stateCond.Lock(); - bool doExit = m_stopping; - m_stateCond.UnLock(); - if (doExit) return 0; - - m_queueCond.Lock(); - - if ( ! m_tasks_queue.empty()) - { - // Exiting with queueMutex held !!! - break; - } - - // returns true on ETIMEDOUT - if ( ! m_queueCond.WaitMS(100)) - { - // Can be empty as result of a signal from destructor - if( ! m_tasks_queue.empty()) - // Exiting with queueMutex held !!! - break; - } - - m_queueCond.UnLock(); - - m_stateCond.Lock(); - doExit = m_stopping; - m_stateCond.UnLock(); - if (doExit) return 0; - - Task* t = CreateTaskForFirstUndownloadedBlock(); - if (t) - return t; - else if (m_cfi.IsComplete()) - return 0; - } - - Task *task = m_tasks_queue.front(); - m_tasks_queue.pop_front(); - - m_queueCond.UnLock(); - - assert(task->ramBlockIdx >=0); - clLog()->Info(XrdCl::AppMsg, "Prefetch::GetNextTask [%d] from queue %s", m_ram.m_blockStates[task->ramBlockIdx].fileBlockIdx, lPath()); - - return task; -} - -//______________________________________________________________________________ -void -Prefetch::DoTask(Task* task) -{ - // read block from client into buffer - int fileBlockIdx = m_ram.m_blockStates[task->ramBlockIdx].fileBlockIdx; - long long offset = fileBlockIdx * m_cfi.GetBufferSize(); - - long long rw_size = m_cfi.GetBufferSize(); - // fix size if this is the last file block - if ( offset + rw_size - m_offset > m_fileSize ) { - rw_size = m_fileSize + m_offset - offset; - assert (rw_size < m_cfi.GetBufferSize()); - } - int missing = rw_size; - int cnt = 0; - char* buff = m_ram.m_buffer; - buff += task->ramBlockIdx * m_cfi.GetBufferSize(); - while (missing) - { - clLog()->Dump(XrdCl::AppMsg, "Prefetch::DoTask() for block f = %d r = %dsingal = %p %s", fileBlockIdx, task->ramBlockIdx, task->condVar, lPath()); - int retval = m_input.Read(buff, offset, missing); - if (retval < 0) - { - clLog()->Warning(XrdCl::AppMsg, "Prefetch::DoTask() failed for negative ret %d block %d %s", retval, fileBlockIdx , lPath()); - break; - } - - missing -= retval; - offset += retval; - buff += retval; - ++cnt; - if (cnt > PREFETCH_MAX_ATTEMPTS) - { - break; - } - } - - m_ram.m_writeMutex.Lock(); - if (missing) { - m_ram.m_blockStates[task->ramBlockIdx].status = kReadFailed; - m_ram.m_blockStates[task->ramBlockIdx].readErrno = errno; - } - else { - m_ram.m_blockStates[task->ramBlockIdx].status = kReadSuccess; - m_ram.m_blockStates[task->ramBlockIdx].readErrno = 0; - } - m_ram.m_writeMutex.Broadcast(); - m_ram.m_writeMutex.UnLock(); - - if (missing == 0) - { - // queue for ram to disk write - XrdSysCondVarHelper monitor(m_stateCond); - if (!m_stopping) { Cache::AddWriteTask(this, task->ramBlockIdx, rw_size, task->condVar ? true : false ); - } - else { - m_ram.m_blockStates[task->ramBlockIdx].refCount--; - } - - } - else - { - DecRamBlockRefCount(task->ramBlockIdx); - clLog()->Dump(XrdCl::AppMsg, "Prefetch::DoTask() incomplete read missing %d for block %d %s", missing, fileBlockIdx, lPath()); - } -} - -//_________________________________________________________________________________________________ -void -Prefetch::WriteBlockToDisk(int ramIdx, size_t size) -{ - // called from XrdFileCache::Cache when process queue - - int fileIdx = m_ram.m_blockStates[ramIdx].fileBlockIdx; - char* buff = m_ram.m_buffer; - buff += ramIdx*m_cfi.GetBufferSize(); - assert(ramIdx >=0 && ramIdx < m_ram.m_numBlocks); - int retval = 0; - - // write block buffer into disk file - long long offset = fileIdx * m_cfi.GetBufferSize() - m_offset; - int buffer_remaining = size; - int buffer_offset = 0; - int cnt = 0; - while ((buffer_remaining > 0) && // There is more to be written - (((retval = m_output->Write(buff, offset + buffer_offset, buffer_remaining)) != -1) - || (errno == EINTR))) // Write occurs without an error - { - buffer_remaining -= retval; - buff += retval; - cnt++; - - if (buffer_remaining) - { - clLog()->Warning(XrdCl::AppMsg, "Prefetch::WriteToDisk() reattempt[%d] writing missing %d for block %d %s", - cnt, buffer_remaining, fileIdx, lPath()); - } - if (cnt > PREFETCH_MAX_ATTEMPTS) - { - clLog()->Error(XrdCl::AppMsg, "Prefetch::WriteToDisk() write failes too manny attempts %s", lPath()); - return; - } - } - - // set bit fetched - clLog()->Dump(XrdCl::AppMsg, "Prefetch::WriteToDisk() success set bit for block [%d] size [%d] %s", fileIdx, size, lPath()); - int pfIdx = fileIdx - m_offset/m_cfi.GetBufferSize(); - m_downloadStatusMutex.Lock(); - m_cfi.SetBitFetched(pfIdx); - m_downloadStatusMutex.UnLock(); - - - // set bit synced - bool schedule_sync = false; - { - XrdSysMutexHelper _lck(&m_syncStatusMutex); - - if (m_in_sync) - { - m_writes_during_sync.push_back(pfIdx); - } - else - { - m_cfi.SetBitWriteCalled(pfIdx); - ++m_non_flushed_cnt; - } - - if (m_non_flushed_cnt >= 100) - { - schedule_sync = true; - m_in_sync = true; - m_non_flushed_cnt = 0; - } - } - - if (schedule_sync) - { - XrdPosixGlobals::schedP->Schedule(m_syncer); - } -} - -//______________________________________________________________________________ -void Prefetch::Sync() -{ - clLog()->Dump(XrdCl::AppMsg, "Prefetch::Sync %s", lPath()); - - m_output->Fsync(); - - m_cfi.WriteHeader(m_infoFile); - - int written_while_in_sync; - { - XrdSysMutexHelper _lck(&m_syncStatusMutex); - - for (std::vector::iterator i = m_writes_during_sync.begin(); i != m_writes_during_sync.end(); ++i) - { - m_cfi.SetBitWriteCalled(*i); - } - written_while_in_sync = m_non_flushed_cnt = (int) m_writes_during_sync.size(); - m_writes_during_sync.clear(); - - m_in_sync = false; - } - - clLog()->Dump(XrdCl::AppMsg, "Prefetch::Sync %d blocks written during sync.", written_while_in_sync); - - m_infoFile->Fsync(); -} - -//______________________________________________________________________________ -void Prefetch::DecRamBlockRefCount(int ramIdx) -{ - clLog()->Dump(XrdCl::AppMsg, "Prefetch::DecRamBlockRefCount %d %d %s", m_ram.m_blockStates[ramIdx].fileBlockIdx, ramIdx,lPath() ); - - // mark ram block available - m_ram.m_writeMutex.Lock(); - assert(m_ram.m_blockStates[ramIdx].refCount); - assert(ramIdx >= 0 && ramIdx < m_ram.m_numBlocks); - - m_ram.m_blockStates[ramIdx].refCount --; - if (m_ram.m_blockStates[ramIdx].refCount == 0) { - m_ram.m_blockStates[ramIdx].fileBlockIdx = -1; - } - m_ram.m_writeMutex.UnLock(); -} - -//______________________________________________________________________________ -bool Prefetch::ReadFromTask(int iFileBlockIdx, char* iBuff, long long iOff, size_t iSize) -{ - // offs == offset inside the block, size read size in block - clLog()->Dump(XrdCl::AppMsg, "Prefetch::ReadFromTask fileIdx= %d begin", iFileBlockIdx); - - m_stateCond.Lock(); - bool doExit = m_stopping; - m_stateCond.UnLock(); - if (doExit) return false; - - if (Cache::HaveFreeWritingSlots()) - { - int ramIdx = -1; - m_ram.m_writeMutex.Lock(); - - int nRR = 0; - for (int i =0 ; i < m_ram.m_numBlocks; ++i) { - if (m_ram.m_blockStates[i].fromRead && m_ram.m_blockStates[i].refCount > 0) nRR++; - } - - if (nRR < Factory::GetInstance().RefConfiguration().m_NRamBuffersRead) { - for (int i =0 ; i < m_ram.m_numBlocks; ++i) - { - if (m_ram.m_blockStates[i].refCount == 0) - { - assert(m_ram.m_blockStates[i].fileBlockIdx == -1); - ramIdx = i; - m_ram.m_blockStates[i].refCount = 1; - m_ram.m_blockStates[i].fileBlockIdx = iFileBlockIdx; - m_ram.m_blockStates[i].fromRead = true; - m_ram.m_blockStates[i].status = kReadWait; - break; - } - } - } - m_ram.m_writeMutex.UnLock(); - - if (ramIdx >= 0) - { - clLog()->Dump(XrdCl::AppMsg, "Prefetch::ReadFromTask, going to add task fileIdx=%d ", iFileBlockIdx); - XrdSysCondVar newTaskCond(0); - { - XrdSysCondVarHelper xx(newTaskCond); - - Task* task = new Task(ramIdx, &newTaskCond); - - m_queueCond.Lock(); - m_tasks_queue.push_front(task); - m_queueCond.Signal(); - m_queueCond.UnLock(); - - clLog()->Dump(XrdCl::AppMsg, "Prefetch::ReadFromTask wait task %p confvar %p", task, task->condVar); - - newTaskCond.Wait(); - } - if (m_ram.m_blockStates[ramIdx].status == kReadSuccess) - { - clLog()->Dump(XrdCl::AppMsg, "Prefetch::ReadFromTask memcpy from RAM to IO::buffer fileIdx=%d ", iFileBlockIdx); - long long inBlockOff = iOff - iFileBlockIdx * m_cfi.GetBufferSize(); - char* srcBuff = m_ram.m_buffer + ramIdx*m_cfi.GetBufferSize(); - memcpy(iBuff, srcBuff + inBlockOff, iSize); - } - else - { - clLog()->Error(XrdCl::AppMsg, "Prefetch::ReadFromTask client fileIdx=%d failed", iFileBlockIdx); - } - - return m_ram.m_blockStates[ramIdx].status == kReadSuccess; - } - else { - clLog()->Debug(XrdCl::AppMsg, "Prefetch::ReadFromTask can't get free ram, not enough resources"); - return false; - } - } - else { - clLog()->Debug(XrdCl::AppMsg, "Prefetch::ReadFromTask write queue full, not enough resources"); - return false; - } -} - -//______________________________________________________________________________ - -ssize_t Prefetch::ReadInBlocks(char *buff, off_t off, size_t size) -{ - long long off0 = off; - int idx_first = off0 / m_cfi.GetBufferSize(); - int idx_last = (off0 + size -1)/ m_cfi.GetBufferSize(); - - size_t bytes_read = 0; - for (int blockIdx = idx_first; blockIdx <= idx_last; ++blockIdx) - { - - int readBlockSize = size; - if (idx_first != idx_last) - { - if (blockIdx == idx_first) - { - readBlockSize = (blockIdx + 1) * m_cfi.GetBufferSize() - off0; - clLog()->Dump(XrdCl::AppMsg, "Read partially till the end of the block %s", lPath()); - } - else if (blockIdx == idx_last) - { - readBlockSize = (off0+size) - blockIdx*m_cfi.GetBufferSize(); - clLog()->Dump(XrdCl::AppMsg, "Read partially from beginning of block %s", lPath()); - } - else - { - readBlockSize = m_cfi.GetBufferSize(); - } - } - - if (readBlockSize > m_cfi.GetBufferSize()) { - clLog()->Error(XrdCl::AppMsg, "block size invalid"); - } - - int retvalBlock = -1; - // now do per block read at Read(buff, off, readBlockSize) - - m_downloadStatusMutex.Lock(); - bool dsl = m_cfi.TestBit(blockIdx - m_offset/m_cfi.GetBufferSize()); - m_downloadStatusMutex.UnLock(); - - if (dsl) - { - retvalBlock = m_output->Read(buff, off - m_offset, readBlockSize); - m_stats.m_BytesDisk += retvalBlock; - clLog()->Dump(XrdCl::AppMsg, "Prefetch::ReadInBlocks [%d] disk = %d",blockIdx, retvalBlock); - } - else - { - int RamIdx = -1; - m_ram.m_writeMutex.Lock(); - for (int ri = 0; ri < m_ram.m_numBlocks; ++ri ) - { - if (m_ram.m_blockStates[ri].fileBlockIdx == blockIdx) - { - RamIdx = ri; - m_ram.m_blockStates[ri].refCount++; - clLog()->Dump(XrdCl::AppMsg, "Prefetch::ReadInBlocks ram = %d file block = %d wait", RamIdx, blockIdx); - while (m_ram.m_blockStates[ri].status == kReadWait) - { - m_ram.m_writeMutex.Wait(); - } - break; - } - } - - m_ram.m_writeMutex.UnLock(); - - if (RamIdx >= 0 ) { - if ( m_ram.m_blockStates[RamIdx].status == kReadSuccess) { - clLog()->Dump(XrdCl::AppMsg, "Prefetch::ReadInBlocks ram = %d file block = %d", RamIdx, blockIdx); - int in_block_off = off - m_ram.m_blockStates[RamIdx].fileBlockIdx *m_cfi.GetBufferSize(); - char *rbuff = m_ram.m_buffer + RamIdx*m_cfi.GetBufferSize() + in_block_off; - memcpy(buff, rbuff, readBlockSize); - DecRamBlockRefCount(RamIdx); - retvalBlock = readBlockSize; - } - else { - errno = m_ram.m_blockStates[RamIdx].readErrno; - DecRamBlockRefCount(RamIdx); - return -1; - } - } - else - { - if (ReadFromTask(blockIdx, buff, off, readBlockSize)) - { - retvalBlock = readBlockSize; // presume since ReadFromTask did not fail, could pass a refrence to ReadFromTask - m_stats.m_BytesRam += retvalBlock; - clLog()->Dump(XrdCl::AppMsg, "Prefetch::ReadInBlocks [%d] fromTask = %d", blockIdx, blockIdx); - } - else - { - retvalBlock = m_input.Read(buff, off, readBlockSize); - clLog()->Dump(XrdCl::AppMsg, "Prefetch::ReadInBlocks [%d] client = %d", blockIdx, retvalBlock); - m_stats.m_BytesMissed += retvalBlock; - } - } - } - - if (retvalBlock > 0 ) - { - bytes_read += retvalBlock; - buff += retvalBlock; - off += retvalBlock; - if (readBlockSize != retvalBlock) - { - clLog()->Warning(XrdCl::AppMsg, "Prefetch::ReadInBlocks incomplete , missing = %d", readBlockSize-retvalBlock); - return bytes_read; - } - } - else - { - return bytes_read; - } - } - return bytes_read; -} - - -//______________________________________________________________________________ - -int Prefetch::ReadV (const XrdOucIOVec *readV, int n) -{ - { - XrdSysCondVarHelper monitor(m_stateCond); - - // AMT check if this can be done once during initalization - if (m_failed) return m_input.ReadV(readV, n); - - if ( ! m_started) - { - m_stateCond.Wait(); - if (m_failed) return 0; - } - } - - // check if read sizes are big enough to cache - - XrdCl::XRootDStatus Status; - XrdCl::ChunkList chunkVec; - XrdCl::VectorReadInfo *vrInfo = 0; - - std::vector cachedReads; - - int nbytes = 0; - for (int i=0; iDebug(XrdCl::AppMsg, "Prefetch::ReadV %d from cache ", i); - if (Read(readV[i].data, readV[i].offset, readV[i].size) < 0) - return -1; - } - else - { - clLog()->Debug(XrdCl::AppMsg, "Prefetch::ReadV %d add back to client vector read ", i); - chunkVec.push_back(XrdCl::ChunkInfo((uint64_t)readV[i].offset, - (uint32_t)readV[i].size, - (void *)readV[i].data - )); - } - - } - if (!chunkVec.empty()) { - XrdCl::File& clFile = ((XrdPosixFile&)m_input).clFile; - Status = clFile.VectorRead(chunkVec, (void *)0, vrInfo); - delete vrInfo; - - if (!Status.IsOK()) - { - XrdPosixMap::Result(Status); - return -1; - } - } - return nbytes; -} -//______________________________________________________________________________ -ssize_t -Prefetch::Read(char *buff, off_t off, size_t size) -{ - { - XrdSysCondVarHelper monitor(m_stateCond); - - // AMT check if this can be done once during initalization - if (m_failed) return m_input.Read(buff, off, size); - - if ( ! m_started) - { - m_stateCond.Wait(); - if (m_failed) return 0; - } - } - - clLog()->Dump(XrdCl::AppMsg, "Prefetch::Read() off = %lld size = %lld. %s", off, size, lPath()); - - bool fileComplete; - m_downloadStatusMutex.Lock(); - fileComplete = m_cfi.IsComplete(); - m_downloadStatusMutex.UnLock(); - - if (fileComplete) - { - int res = m_output->Read(buff, off - m_offset, size); - m_stats.m_BytesDisk += res; - return res; - } - else - { - return ReadInBlocks(buff, off, size); - } -} - - -//______________________________________________________________________________ -void Prefetch::AppendIOStatToFileInfo() -{ - // lock in case several IOs want to write in *cinfo file - m_downloadStatusMutex.Lock(); - if (m_infoFile) - { - Info::AStat as; - as.DetachTime = time(0); - as.BytesDisk = m_stats.m_BytesDisk; - as.BytesRam = m_stats.m_BytesRam; - as.BytesMissed = m_stats.m_BytesMissed; - m_cfi.AppendIOStat(as, (XrdOssDF*)m_infoFile); - } - else - { - clLog()->Warning(XrdCl::AppMsg, "Prefetch::AppendIOStatToFileInfo() info file not opened %s", lPath()); - } - m_downloadStatusMutex.UnLock(); -} - - - diff --git a/src/XrdFileCache/XrdFileCachePrefetch.hh b/src/XrdFileCache/XrdFileCachePrefetch.hh deleted file mode 100644 index f1cc467d761..00000000000 --- a/src/XrdFileCache/XrdFileCachePrefetch.hh +++ /dev/null @@ -1,199 +0,0 @@ -#ifndef __XRDFILECACHE_PREFETCH_HH__ -#define __XRDFILECACHE_PREFETCH_HH__ -//---------------------------------------------------------------------------------- -// Copyright (c) 2014 by Board of Trustees of the Leland Stanford, Jr., University -// Author: Alja Mrak-Tadel, Matevz Tadel, Brian Bockelman -//---------------------------------------------------------------------------------- -// XRootD is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// XRootD is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with XRootD. If not, see . -//---------------------------------------------------------------------------------- - -#include -#include - -#include "XrdCl/XrdClDefaultEnv.hh" - -#include "XrdFileCacheInfo.hh" -#include "XrdFileCacheStats.hh" - -class XrdJob; -class XrdOucIOVec; - -namespace XrdCl -{ - class Log; -} - -namespace XrdFileCache -{ - //---------------------------------------------------------------------------- - //! Downloads data into a file on local disk and handles IO read requests. - //---------------------------------------------------------------------------- - class Prefetch - { - friend class IOEntireFile; - friend class IOFileBlock; - enum ReadRamState_t { kReadWait, kReadSuccess, kReadFailed}; - - struct Task; - public: - //------------------------------------------------------------------------ - //! Constructor. - //------------------------------------------------------------------------ - Prefetch(XrdOucCacheIO& inputFile, std::string& path, - long long offset, long long fileSize); - - //------------------------------------------------------------------------ - //! Destructor. - //------------------------------------------------------------------------ - ~Prefetch(); - - //--------------------------------------------------------------------- - //! Thread function for file prefetching. - //--------------------------------------------------------------------- - void Run(); - - //---------------------------------------------------------------------- - //! Reference to prefetch statistics. - //---------------------------------------------------------------------- - Stats& GetStats() { return m_stats; } - - //---------------------------------------------------------------------- - //! Write block to file on disk. Called from Cache. - //---------------------------------------------------------------------- - void WriteBlockToDisk(int ramIdx, size_t size); - - //---------------------------------------------------------------------- - //! Decrease block reference count. - //---------------------------------------------------------------------- - void DecRamBlockRefCount(int ramIdx); - - //---------------------------------------------------------------------- - //! \brief Initiate close. Return true if still IO active. - //! Used in XrdPosixXrootd::Close() - //---------------------------------------------------------------------- - bool InitiateClose(); - - //---------------------------------------------------------------------- - //! Sync file cache inf o and output data with disk - //---------------------------------------------------------------------- - void Sync(); - - - protected: - //! Read from disk, RAM, task, or client. - ssize_t Read(char * buff, off_t offset, size_t size); - - //! Vector read from disk if block is already downloaded, else ReadV from client. - int ReadV (const XrdOucIOVec *readV, int n); - - //! Write cache statistics in *cinfo file. - void AppendIOStatToFileInfo(); - - private: - //---------------------------------------------------------------------- - //! A prefetching task -- a file region that requires preferential treatment. - //---------------------------------------------------------------------- - struct Task - { - int ramBlockIdx; //!< idx in the in-memory buffer - XrdSysCondVar *condVar; //!< signal when complete - - Task(): ramBlockIdx(-1), condVar(0) {} - Task(int r, XrdSysCondVar *cv): - ramBlockIdx(r), condVar(cv) {} - ~Task() {} - }; - - struct RAMBlock { - int fileBlockIdx; //!< offset in output file - int refCount; //!< read and write reference count - bool fromRead; //!< is ram requested from prefetch or read - ReadRamState_t status; //!< read from client status - int readErrno; //!< posix error on read fail - - RAMBlock():fileBlockIdx(-1), refCount(0), fromRead(false), status(kReadWait) {} - }; - - struct RAM - { - int m_numBlocks; //!< number of in memory blocks - char* m_buffer; //!< buffer m_numBlocks x size_of_block - RAMBlock* m_blockStates; //!< referenced structure - XrdSysCondVar m_writeMutex; //!< write mutex - - RAM(); - ~RAM(); - }; - - //! Stop Run thread. - void CloseCleanly(); - - //! Get blocks to prefetch. - Task* GetNextTask(); - - //! Open file handle for data file and info file on local disk. - bool Open(); - - //! Short log alias. - XrdCl::Log* clLog() const { return XrdCl::DefaultEnv::GetLog(); } - - //! Split read in blocks. - ssize_t ReadInBlocks( char* buff, off_t offset, size_t size); - - //! Prefetch block. - Task* CreateTaskForFirstUndownloadedBlock(); - - //! Create task from read request and wait its completed. - bool ReadFromTask(int bIdx, char* buff, long long off, size_t size); - - //! Read from client into in memory cache, queue ram buffer for disk write. - void DoTask(Task* task); - - //! Log path - const char* lPath() const; - - RAM m_ram; //!< in memory cache - - XrdOssDF *m_output; //!< file handle for data file on disk - XrdOssDF *m_infoFile; //!< file handle for data-info file on disk - Info m_cfi; //!< download status of file blocks and access statistics - XrdOucCacheIO &m_input; //!< original data source - - std::string m_temp_filename; //!< filename of data file on disk - - long long m_offset; //!< offset of disk file for block-based operation - long long m_fileSize; //!< size of disk file for block-based operation - - bool m_started; //!< state of run thread - bool m_failed; //!< reading from original source or writing to disk has failed - bool m_stopping; //!< run thread should be stopped - bool m_stopped; //!< prefetch is stopped - XrdSysCondVar m_stateCond; //!< state condition variable - - XrdSysMutex m_downloadStatusMutex; //!< mutex locking access to m_cfi object - - std::deque m_tasks_queue; //!< download queue - XrdSysCondVar m_queueCond; //!< m_tasks_queue condition variable - - Stats m_stats; //!< cache statistics, used in IO detach - - // fsync - XrdSysMutex m_syncStatusMutex; //!< mutex locking fsync status - XrdJob *m_syncer; - std::vector m_writes_during_sync; - int m_non_flushed_cnt; - bool m_in_sync; - }; -} -#endif diff --git a/src/XrdFileCache/XrdFileCachePrint.cc b/src/XrdFileCache/XrdFileCachePrint.cc index a72da409b3c..a87fb5956d5 100644 --- a/src/XrdFileCache/XrdFileCachePrint.cc +++ b/src/XrdFileCache/XrdFileCachePrint.cc @@ -71,8 +71,9 @@ void Print::printFile(const std::string& path) int cntd = 0; for (int i = 0; i < cfi.GetSizeInBits(); ++i) if (cfi.TestBit(i)) cntd++; - - printf("version == %d, bufferSize %lld nBlocks %d nDownlaoded %d %s\n",cfi.GetVersion(), cfi.GetBufferSize(), cfi.GetSizeInBits() , cntd, (cfi.GetSizeInBits() == cntd) ? " complete" :""); + printf("version == %d, bufferSize %lld nBlocks %d nDownloaded %d %s\n", + cfi.GetVersion(), cfi.GetBufferSize(), cfi.GetSizeInBits(), cntd, + (cfi.GetSizeInBits() == cntd) ? "complete" : ""); if (m_verbose) { printf("printing %d blocks: \n", cfi.GetSizeInBits()); diff --git a/src/XrdFileCache/XrdFileCachePurge.cc b/src/XrdFileCache/XrdFileCachePurge.cc new file mode 100644 index 00000000000..19ed5a3dd3d --- /dev/null +++ b/src/XrdFileCache/XrdFileCachePurge.cc @@ -0,0 +1,207 @@ +#include "XrdFileCache.hh" + +using namespace XrdFileCache; + +#include +#include "XrdOuc/XrdOucEnv.hh" + +namespace +{ + class FPurgeState + { + public: + struct FS + { + std::string path; + long long nByte; + + FS(const char* p, long long n) : path(p), nByte(n) {} + }; + + typedef std::multimap map_t; + typedef map_t::iterator map_i; + + FPurgeState(long long iNByteReq) : nByteReq(iNByteReq), nByteAccum(0) {} + + map_t fmap; + + void checkFile (time_t iTime, const char* iPath, long long iNByte) + { + if (nByteAccum < nByteReq || iTime < fmap.rbegin()->first) + { + fmap.insert(std::pair (iTime, FS(iPath, iNByte))); + nByteAccum += iNByte; + + // remove newest files from map if necessary + while (nByteAccum > nByteReq) + { + time_t nt = fmap.begin()->first; + std::pair ret = fmap.equal_range(nt); + for (map_i it2 = ret.first; it2 != ret.second; ++it2) + nByteAccum -= it2->second.nByte; + fmap.erase(ret.first, ret.second); + } + } + } + + private: + long long nByteReq; + long long nByteAccum; + }; +} + +void FillFileMapRecurse( XrdOssDF* iOssDF, const std::string& path, FPurgeState& purgeState) +{ + char buff[256]; + XrdOucEnv env; + int rdr; + const size_t InfoExtLen = strlen(XrdFileCache::Info::m_infoExtension); // cached var + XrdCl::Log *log = XrdCl::DefaultEnv::GetLog(); + + Cache& factory = Cache::GetInstance(); + while ((rdr = iOssDF->Readdir(&buff[0], 256)) >= 0) + { + // printf("readdir [%s]\n", buff); + std::string np = path + "/" + std::string(buff); + size_t fname_len = strlen(&buff[0]); + if (fname_len == 0) + { + // std::cout << "Finish read dir.[" << np <<"] Break loop \n"; + break; + } + + if (strncmp("..", &buff[0], 2) && strncmp(".", &buff[0], 1)) + { + XrdOssDF* dh = factory.GetOss()->newDir(factory.RefConfiguration().m_username.c_str()); + XrdOssDF* fh = factory.GetOss()->newFile(factory.RefConfiguration().m_username.c_str()); + + if (fname_len > InfoExtLen && strncmp(&buff[fname_len - InfoExtLen ], XrdFileCache::Info::m_infoExtension, InfoExtLen) == 0) + { + // XXXX MT - shouldn't we also check if it is currently opened? + + fh->Open(np.c_str(), O_RDONLY, 0600, env); + Info cinfo(factory.RefConfiguration().m_bufferSize); + time_t accessTime; + cinfo.Read(fh); + if (cinfo.GetLatestDetachTime(accessTime, fh)) + { + log->Debug(XrdCl::AppMsg, "FillFileMapRecurse() checking %s accessTime %d ", buff, (int)accessTime); + purgeState.checkFile(accessTime, np.c_str(), cinfo.GetNDownloadedBytes()); + } + else + { + // cinfo file does not contain any known accesses, use stat.mtime instead. + + log->Info(XrdCl::AppMsg, "FillFileMapRecurse() could not get access time for %s, trying stat.\n", np.c_str()); + + XrdOss* oss = Cache::GetInstance().GetOss(); + struct stat fstat; + + if (oss->Stat(np.c_str(), &fstat) == XrdOssOK) + { + accessTime = fstat.st_mtime; + log->Info(XrdCl::AppMsg, "FillFileMapRecurse() determined access time for %s via stat: %lld\n", + np.c_str(), accessTime); + + purgeState.checkFile(accessTime, np.c_str(), cinfo.GetNDownloadedBytes()); + } + else + { + // This really shouldn't happen ... but if it does remove cinfo and the data file right away. + + log->Warning(XrdCl::AppMsg, "FillFileMapRecurse() could not get access time for %s. Purging directly.\n", + np.c_str()); + + oss->Unlink(np.c_str()); + np = np.substr(0, np.size() - strlen(XrdFileCache::Info::m_infoExtension)); + oss->Unlink(np.c_str()); + } + } + } + else if (dh->Opendir(np.c_str(), env) >= 0) + { + FillFileMapRecurse(dh, np, purgeState); + } + + delete dh; dh = 0; + delete fh; fh = 0; + } + } +} + +void Cache::CacheDirCleanup() +{ + // check state every sleep seconds + const static int sleept = 300; + struct stat fstat; + XrdOucEnv env; + + XrdOss* oss = Cache::GetInstance().GetOss(); + XrdOssVSInfo sP; + + while (1) + { + // get amount of space to erase + long long bytesToRemove = 0; + if (oss->StatVS(&sP, "public", 1) < 0) + { + clLog()->Error(XrdCl::AppMsg, "Cache::CacheDirCleanup() can't get statvs for dir [%s] \n", m_configuration.m_cache_dir.c_str()); + exit(1); + } + else + { + long long ausage = sP.Total - sP.Free; + clLog()->Info(XrdCl::AppMsg, "Cache::CacheDirCleanup() occupates disk space == %lld", ausage); + if (ausage > m_configuration.m_diskUsageHWM) + { + bytesToRemove = ausage - m_configuration.m_diskUsageLWM; + clLog()->Info(XrdCl::AppMsg, "Cache::CacheDirCleanup() need space for %lld bytes", bytesToRemove); + } + } + + if (bytesToRemove > 0) + { + // make a sorted map of file patch by access time + XrdOssDF* dh = oss->newDir(m_configuration.m_username.c_str()); + if (dh->Opendir(m_configuration.m_cache_dir.c_str(), env) >= 0) + { + FPurgeState purgeState(bytesToRemove * 5 / 4); // prepare 20% more volume than required + + FillFileMapRecurse(dh, m_configuration.m_cache_dir, purgeState); + + // loop over map and remove files with highest value of access time + for (FPurgeState::map_i it = purgeState.fmap.begin(); it != purgeState.fmap.end(); ++it) + { + // XXXX MT - shouldn't we re-check if the file is currently opened? + + std::string path = it->second.path; + // remove info file + if (oss->Stat(path.c_str(), &fstat) == XrdOssOK) + { + bytesToRemove -= fstat.st_size; + oss->Unlink(path.c_str()); + clLog()->Info(XrdCl::AppMsg, "Cache::CacheDirCleanup() removed %s size %lld", + path.c_str(), fstat.st_size); + } + + // remove data file + path = path.substr(0, path.size() - strlen(XrdFileCache::Info::m_infoExtension)); + if (oss->Stat(path.c_str(), &fstat) == XrdOssOK) + { + bytesToRemove -= it->second.nByte; + oss->Unlink(path.c_str()); + clLog()->Info(XrdCl::AppMsg, "Cache::CacheDirCleanup() removed %s bytes %lld, stat_size %lld", + path.c_str(), it->second.nByte, fstat.st_size); + } + + if (bytesToRemove <= 0) + break; + } + } + dh->Close(); + delete dh; dh =0; + } + + sleep(sleept); + } +} diff --git a/src/XrdFileCache/XrdFileCacheVRead.cc b/src/XrdFileCache/XrdFileCacheVRead.cc new file mode 100644 index 00000000000..cbfdb3ae471 --- /dev/null +++ b/src/XrdFileCache/XrdFileCacheVRead.cc @@ -0,0 +1,321 @@ + +#include "XrdFileCacheFile.hh" +#include "XrdFileCacheFactory.hh" +#include "XrdFileCache.hh" + +#include "XrdFileCacheInfo.hh" +#include "XrdFileCacheStats.hh" + +#include "XrdOss/XrdOss.hh" +#include "XrdCl/XrdClDefaultEnv.hh" +#include "XrdCl/XrdClFile.hh" +#include "XrdCl/XrdClXRootDResponses.hh" +#include "XrdPosix/XrdPosixFile.hh" + +namespace XrdFileCache { + + // a list of IOVec chuncks that match a given block index + // first element is block index, the following vector elements are chunk readv indicies + struct ReadVChunkListDisk { + ReadVChunkListDisk(int i) : block_idx(i) {}; + int block_idx; + std::vector arr; + }; + + struct ReadVChunkListRAM { + ReadVChunkListRAM(Block*b, std::vector * iarr) : block(b), arr(iarr) { }; + Block* block; + std::vector * arr; + }; + + // RAM + struct ReadVBlockListRAM { + std::vector bv; + bool AddEntry(Block* block, int chunkIdx) + { + for ( std::vector::iterator i = bv.begin(); i != bv.end(); ++i) + { + if (i->block == block) { + i->arr->push_back(chunkIdx); + return false; + } + } + bv.push_back(ReadVChunkListRAM(block, new std::vector)); + bv.back().arr->push_back(chunkIdx); + return true; + } + }; + + // Disk + struct ReadVBlockListDisk { + std::vector bv; + void AddEntry(int blockIdx, int chunkIdx) + { + for ( std::vector::iterator i = bv.begin(); i != bv.end(); ++i) + { + if (i->block_idx == blockIdx) { + i->arr.push_back(chunkIdx); + return; + } + } + bv.push_back(XrdFileCache::ReadVChunkListDisk(blockIdx)); + bv.back().arr.push_back(chunkIdx); + } + }; +} + +using namespace XrdFileCache; + +//------------------------------------- + +int File::ReadV (const XrdOucIOVec *readV, int n) +{ + clLog()->Debug(XrdCl::AppMsg, "ReadV for %d chunks", n); + + int bytesRead = 0; + + ReadVBlockListRAM blocks_to_process; + std::vector blks_processed; + ReadVBlockListDisk blocks_on_disk; + XrdCl::ChunkList chunkVec; + DirectResponseHandler* direct_handler = 0; + if (!VReadPreProcess(readV, n, blocks_to_process, blocks_on_disk, chunkVec)) + bytesRead = -1; + + + // issue a client read + + if (bytesRead >= 0) { + if (!chunkVec.empty()) { + direct_handler = new DirectResponseHandler(1); + XrdCl::File &client = ((XrdPosixFile*)(&m_input))->clFile; + // TODO check interface in the client file + XrdCl::XRootDStatus vrStatus = client.VectorRead(chunkVec, (void*) 0, direct_handler); + if (!vrStatus.IsOK()) { + bytesRead = -1; + } + } + } + + // disk read + if (bytesRead >= 0) { + int dr = VReadFromDisk(readV, n, blocks_on_disk); + if (dr < 0) + bytesRead = dr; + else + bytesRead += dr; + } + + // read from cached blocks + if (bytesRead >=0) { + int br = VReadProcessBlocks(readV, n, blocks_to_process.bv, blks_processed); + if (br < 0) + bytesRead = br; + else + bytesRead += br; + } + + + // check direct requests have arrived, get bytes read from read handle + if (bytesRead >=0 && direct_handler != 0) + { + XrdSysCondVarHelper _lck(direct_handler->m_cond); + if (direct_handler->m_to_wait == 1) + { + direct_handler->m_cond.Wait(); + } + + if (direct_handler->m_errno == 0) + { + for (XrdCl::ChunkList::iterator i = chunkVec.begin(); i != chunkVec.end(); ++i) + bytesRead += i->length; + } + else + { + errno = direct_handler->m_errno; + bytesRead = -1; + } + } + + + { + XrdSysCondVarHelper _lck(m_downloadCond); + + // decrease ref count on the remaining blocks + // this happens in case read process has been broke due to previous errors + for (std::vector::iterator i = blocks_to_process.bv.begin(); i != blocks_to_process.bv.end(); ++i) + dec_ref_count(i->block); + + for (std::vector::iterator i = blks_processed.begin(); i != blks_processed.end(); ++i) + dec_ref_count(i->block); + } + + // remove objects on heap + delete direct_handler; + for (std::vector::iterator i = blocks_to_process.bv.begin(); i != blocks_to_process.bv.end(); ++i) + delete i->arr; + for (std::vector::iterator i = blks_processed.begin(); i != blks_processed.end(); ++i) + delete i->arr; + + + clLog()->Debug(XrdCl::AppMsg, "VRead exit, total = %d", bytesRead); + return bytesRead; +} + +//______________________________________________________________________________ + + +bool File::VReadPreProcess(const XrdOucIOVec *readV, int n, ReadVBlockListRAM& blocks_to_process, ReadVBlockListDisk& blocks_on_disk, XrdCl::ChunkList& chunkVec) +{ + XrdSysCondVarHelper _lck(m_downloadCond); + for (int iov_idx=0; iov_idxDebug(XrdCl::AppMsg, "VReadPreProcess chunk %lld@%lld", readV[iov_idx].size, readV[iov_idx].offset); + BlockMap_i bi = m_block_map.find(block_idx); + if (bi != m_block_map.end()) + { + if ( blocks_to_process.AddEntry(bi->second, iov_idx)) + inc_ref_count(bi->second); + clLog()->Debug(XrdCl::AppMsg, "VReadPreProcess block %d in map", block_idx); + } + else if (m_cfi.TestBit(block_idx)) + { + blocks_on_disk.AddEntry(block_idx, iov_idx); + clLog()->Debug(XrdCl::AppMsg, "VReadPreProcess block %d , chunk idx = %d on disk", block_idx,iov_idx ); + } + else { + if ( Factory::GetInstance().GetCache()->HaveFreeWritingSlots() && Factory::GetInstance().GetCache()->RequestRAMBlock()) + { + Block *b = RequestBlock(block_idx, false); + if (!b) return false; + blocks_to_process.AddEntry(b, iov_idx); + clLog()->Debug(XrdCl::AppMsg, "VReadPreProcess request block %d", block_idx); + inc_ref_count(b); + } + else { + long long off; // offset in user buffer + long long blk_off; // offset in block + long long size; // size to copy + const long long BS = m_cfi.GetBufferSize(); + overlap(block_idx, BS, readV[iov_idx].offset, readV[iov_idx].size, off, blk_off, size); + chunkVec.push_back(XrdCl::ChunkInfo( BS*block_idx + blk_off,size, readV[iov_idx].data+off)); + clLog()->Debug(XrdCl::AppMsg, "VReadPreProcess direct read %d", block_idx); + } + } + } + } + + return true; +} + + +//----------------------------------------------------------------------------------- + +int File::VReadFromDisk(const XrdOucIOVec *readV, int n, ReadVBlockListDisk& blocks_on_disk) +{ + int bytes_read = 0; + for (std::vector::iterator bit = blocks_on_disk.bv.begin(); bit != blocks_on_disk.bv.end(); ++bit ) + { + int blockIdx = bit->block_idx; + for (std::vector::iterator chunkIt = bit->arr.begin(); chunkIt != bit->arr.end(); ++chunkIt) + { + int chunkIdx = *chunkIt; + + long long off; // offset in user buffer + long long blk_off; // offset in block + long long size; // size to copy + + + clLog()->Debug(XrdCl::AppMsg, "VReadFromDisk block=%d chunk=%d", blockIdx, chunkIdx); + overlap(blockIdx, m_cfi.GetBufferSize(), readV[chunkIdx].offset, readV[chunkIdx].size, off, blk_off, size); + + int rs = m_output->Read(readV[chunkIdx].data + off, blockIdx*m_cfi.GetBufferSize() + blk_off, size); + if (rs >=0 ) { + bytes_read += rs; + } + else { + // ofs read shoul set the errno + + clLog()->Error(XrdCl::AppMsg, "VReadFromDisk FAILED block=%d chunk=%d off=%lld, blk_off=%lld, size=%lld, chunfoff =%lld", blockIdx, chunkIdx, off, blk_off, size,readV[chunkIdx].offset ); + + return -1; + } + } + } + + return bytes_read; +} + +//______________________________________________________________________________ + + +int File::VReadProcessBlocks(const XrdOucIOVec *readV, int n, + std::vector& blocks_to_process, std::vector& blocks_processed) +{ + int bytes_read = 0; + while ( (! blocks_to_process.empty()) && (bytes_read >= 0)) + { + std::vector finished; + { + XrdSysCondVarHelper _lck(m_downloadCond); + std::vector::iterator bi = blocks_to_process.begin(); + while (bi != blocks_to_process.end()) + { + if (bi->block->is_finished()) + { + finished.push_back(ReadVChunkListRAM(bi->block, bi->arr)); + // std::vector::iterator bj = bi++; + blocks_to_process.erase(bi); + } + else + { + ++bi; + } + } + + if (finished.empty()) + { + m_downloadCond.Wait(); + continue; + } + } + + + std::vector::iterator bi = finished.begin(); + while (bi != finished.end()) + { + if (bi->block->is_ok()) { + + for ( std::vector::iterator chunkIt = bi->arr->begin(); chunkIt < bi->arr->end(); ++chunkIt ) { + long long off; // offset in user buffer + long long blk_off; // offset in block + long long size; // size to copy + + int block_idx =bi->block->m_offset/m_cfi.GetBufferSize(); + overlap(block_idx, m_cfi.GetBufferSize(), readV[*chunkIt].offset, readV[*chunkIt].size, off, blk_off, size); + memcpy(readV[*chunkIt].data + off, &(bi->block->m_buff[blk_off]), size); + bytes_read += size; + } + } + else { + bytes_read = -1; + errno = bi->block->m_errno; + break; + } + + ++bi; + } + + // add finished to processed list + std::copy(finished.begin(), finished.end(), std::back_inserter(blocks_processed)); + finished.clear(); + } + + clLog()->Info(XrdCl::AppMsg, "VReadProcessBlocks total read == %d", bytes_read); + return bytes_read; +}