From 24899eb67b882342c96cfd701e17c40fb61f12a1 Mon Sep 17 00:00:00 2001 From: Matevz Tadel Date: Mon, 16 Nov 2020 15:49:43 -0800 Subject: [PATCH 1/3] Add XrdOssDir::getFD() needed for XrdOssAt commands. --- src/XrdOss/XrdOssApi.hh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/XrdOss/XrdOssApi.hh b/src/XrdOss/XrdOssApi.hh index fc001cd5d33..d2451079d0d 100644 --- a/src/XrdOss/XrdOssApi.hh +++ b/src/XrdOss/XrdOssApi.hh @@ -55,6 +55,7 @@ int Close(long long *retsz=0); int Opendir(const char *, XrdOucEnv &); int Readdir(char *buff, int blen); int StatRet(struct stat *buff); +int getFD() {return fd;} // Constructor and destructor XrdOssDir(const char *tid, DIR *dP=0) From cc6663c59e66a819b1c8c520b4af3cfadedef05d Mon Sep 17 00:00:00 2001 From: Matevz Tadel Date: Mon, 16 Nov 2020 15:50:38 -0800 Subject: [PATCH 2/3] Initialize collection time variables. --- src/XrdXrootd/XrdXrootdGSReal.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/XrdXrootd/XrdXrootdGSReal.cc b/src/XrdXrootd/XrdXrootdGSReal.cc index 5aa04cce26d..0028684c1bd 100644 --- a/src/XrdXrootd/XrdXrootdGSReal.cc +++ b/src/XrdXrootd/XrdXrootdGSReal.cc @@ -119,6 +119,8 @@ XrdXrootdGSReal::XrdXrootdGSReal(const XrdXrootdGSReal::GSParms &gsParms, udpBFirst = udpBNext = udpBuffer + hdrLen; udpBEnd = udpBuffer + maxL - 1; + tBeg = tEnd = afTime = 0; + // Initialize remaining variables // monType = gsParms.Mode; From e8c70282835e5c4d87f4ee974dec9c06decfffe5 Mon Sep 17 00:00:00 2001 From: Matevz Tadel Date: Wed, 18 Nov 2020 16:07:51 -0800 Subject: [PATCH 3/3] [XCache] Support pgRead and checksums. - Checksums are supported for network. - Scaffolding is done for XrdOssPgi -- on disk storage of checksums. - Use OssAt during purge traversal to reduce pressure on the filesystem. - Report remote sources in g-stream file_close record. --- src/XrdPfc.cmake | 2 + src/XrdPfc/XrdPfc.cc | 158 ++++++------ src/XrdPfc/XrdPfc.hh | 69 ++---- src/XrdPfc/XrdPfcCommand.cc | 2 +- src/XrdPfc/XrdPfcConfiguration.cc | 198 ++++++++++----- src/XrdPfc/XrdPfcFile.cc | 293 +++++++++++++++------- src/XrdPfc/XrdPfcFile.hh | 30 ++- src/XrdPfc/XrdPfcIO.cc | 12 +- src/XrdPfc/XrdPfcIO.hh | 19 +- src/XrdPfc/XrdPfcIOEntireFile.cc | 35 +-- src/XrdPfc/XrdPfcIOEntireFile.hh | 5 +- src/XrdPfc/XrdPfcIOFileBlock.cc | 64 +++-- src/XrdPfc/XrdPfcIOFileBlock.hh | 4 +- src/XrdPfc/XrdPfcInfo.cc | 379 +++++++++++++++++++--------- src/XrdPfc/XrdPfcInfo.hh | 94 +++++-- src/XrdPfc/XrdPfcPrint.cc | 25 +- src/XrdPfc/XrdPfcPurge.cc | 398 +++++++++++++++--------------- src/XrdPfc/XrdPfcTrace.hh | 13 +- src/XrdPfc/XrdPfcTypes.hh | 32 +++ src/XrdPfc/XrdPfcVRead.cc | 12 +- 20 files changed, 1134 insertions(+), 710 deletions(-) create mode 100644 src/XrdPfc/XrdPfcTypes.hh diff --git a/src/XrdPfc.cmake b/src/XrdPfc.cmake index b6ecfa838ad..b00d6b1eac5 100644 --- a/src/XrdPfc.cmake +++ b/src/XrdPfc.cmake @@ -17,6 +17,7 @@ set( LIB_XRD_BLACKLIST XrdBlacklistDecision-${PLUGIN_VERSION} ) add_library( ${LIB_XRD_FILECACHE} MODULE + XrdPfc/XrdPfcTypes.hh XrdPfc/XrdPfc.cc XrdPfc/XrdPfc.hh XrdPfc/XrdPfcConfiguration.cc XrdPfc/XrdPfcPurge.cc @@ -69,6 +70,7 @@ set_target_properties( add_executable( xrdpfc_print XrdPfc/XrdPfcPrint.hh XrdPfc/XrdPfcPrint.cc + XrdPfc/XrdPfcTypes.hh XrdPfc/XrdPfcInfo.hh XrdPfc/XrdPfcInfo.cc) target_link_libraries( diff --git a/src/XrdPfc/XrdPfc.cc b/src/XrdPfc/XrdPfc.cc index 5b29017b7c4..3da53ea8baa 100644 --- a/src/XrdPfc/XrdPfc.cc +++ b/src/XrdPfc/XrdPfc.cc @@ -152,11 +152,9 @@ Cache &Cache::CreateInstance(XrdSysLogger *logger, XrdOucEnv *env) return *m_instance; } -Cache &Cache::GetInstance() -{ - assert (m_instance != 0); - return *m_instance; -} + Cache& Cache::GetInstance() { return *m_instance; } +const Cache& Cache::TheOne() { return *m_instance; } +const Configuration& Cache::Conf() { return m_instance->RefConfiguration(); } bool Cache::Decide(XrdOucCacheIO* io) { @@ -184,7 +182,7 @@ Cache::Cache(XrdSysLogger *logger, XrdOucEnv *env) : m_env(env), m_log(logger, "XrdPfc_"), m_trace(new XrdSysTrace("XrdPfc", logger)), - m_traceID("Manager"), + m_traceID("Cache"), m_oss(0), m_gstream(0), m_prefetch_condVar(0), @@ -192,9 +190,7 @@ Cache::Cache(XrdSysLogger *logger, XrdOucEnv *env) : m_RAM_used(0), m_RAM_write_queue(0), m_RAM_std_size(0), - m_csUVKeep(-1), m_isClient(false), - m_csChk(csChk_Net|csChk_TLS), m_in_purge(false), m_active_cond(0), m_stats_n_purge_cond(0), @@ -209,7 +205,7 @@ Cache::Cache(XrdSysLogger *logger, XrdOucEnv *env) : XrdOucCacheIO *Cache::Attach(XrdOucCacheIO *io, int Options) { - const char* tpfx = "Cache::Attach() "; + const char* tpfx = "Attach() "; if (Cache::GetInstance().Decide(io)) { @@ -219,11 +215,11 @@ XrdOucCacheIO *Cache::Attach(XrdOucCacheIO *io, int Options) if (Cache::GetInstance().RefConfiguration().m_hdfsmode) { - cio = new IOFileBlock(io, m_ouc_stats, *this); + cio = new IOFileBlock(io, *this); } else { - IOEntireFile *ioef = new IOEntireFile(io, m_ouc_stats, *this); + IOEntireFile *ioef = new IOEntireFile(io, *this); if ( ! ioef->HasFile()) { @@ -251,7 +247,7 @@ XrdOucCacheIO *Cache::Attach(XrdOucCacheIO *io, int Options) void Cache::AddWriteTask(Block* b, bool fromRead) { - TRACE(Dump, "Cache::AddWriteTask() bOff=%ld " << b->m_offset); + TRACE(Dump, "AddWriteTask() bOff=" << b->m_offset); { XrdSysMutexHelper lock(&m_RAM_mutex); @@ -279,7 +275,7 @@ void Cache::RemoveWriteQEntriesFor(File *file) { if ((*i)->m_file == file) { - TRACE(Dump, "Cache::Remove entries for " << (void*)(*i) << " path " << file->lPath()); + TRACE(Dump, "Remove entries for " << (void*)(*i) << " path " << file->lPath()); std::list::iterator j = i++; removed_blocks.push_back(*j); sum_size += (*j)->get_size(); @@ -328,7 +324,7 @@ void Cache::ProcessWriteTasks() blks_to_write[bi] = block; - TRACE(Dump, "Cache::ProcessWriteTasks for block " << (void*)(block) << " path " << block->m_file->lPath()); + TRACE(Dump, "ProcessWriteTasks for block " << (void*)(block) << " path " << block->m_file->lPath()); } m_writeQ.size -= n_pushed; @@ -412,7 +408,7 @@ File* Cache::GetFile(const std::string& path, IO* io, long long off, long long f { // Called from virtual IO::Attach - TRACE(Debug, "Cache::GetFile " << path << ", io " << io); + TRACE(Debug, "GetFile " << path << ", io " << io); ActiveMap_i it; @@ -452,10 +448,10 @@ File* Cache::GetFile(const std::string& path, IO* io, long long off, long long f int res = io->Fstat(st); if (res < 0) { errno = res; - TRACE(Error, "Cache::GetFile, could not get valid stat"); + TRACE(Error, "GetFile, could not get valid stat"); } else if (res > 0) { errno = ENOTSUP; - TRACE(Error, "Cache::GetFile, stat returned positive value, this should NOT happen here"); + TRACE(Error, "GetFile, stat returned positive value, this should NOT happen here"); } else { filesize = st.st_size; } @@ -493,7 +489,7 @@ void Cache::ReleaseFile(File* f, IO* io) { // Called from virtual IO::DetachFinalize. - TRACE(Debug, "Cache::ReleaseFile " << f->GetLocalPath() << ", io " << io); + TRACE(Debug, "ReleaseFile " << f->GetLocalPath() << ", io " << io); { XrdSysCondVarHelper lock(&m_active_cond); @@ -575,7 +571,7 @@ void Cache::inc_ref_cnt(File* f, bool lock, bool high_debug) int rc = f->inc_ref_cnt(); if (lock) m_active_cond.UnLock(); - TRACE_INT(tlvl, "Cache::inc_ref_cnt " << f->GetLocalPath() << ", cnt at exit = " << rc); + TRACE_INT(tlvl, "inc_ref_cnt " << f->GetLocalPath() << ", cnt at exit = " << rc); } void Cache::dec_ref_cnt(File* f, bool high_debug) @@ -597,13 +593,13 @@ void Cache::dec_ref_cnt(File* f, bool high_debug) if (cnt == 1) { - TRACE_INT(tlvl, "Cache::dec_ref_cnt " << f->GetLocalPath() << " is in shutdown, ref_cnt = " << cnt + TRACE_INT(tlvl, "dec_ref_cnt " << f->GetLocalPath() << " is in shutdown, ref_cnt = " << cnt << " -- deleting File object without further ado"); delete f; } else { - TRACE_INT(tlvl, "Cache::dec_ref_cnt " << f->GetLocalPath() << " is in shutdown, ref_cnt = " << cnt + TRACE_INT(tlvl, "dec_ref_cnt " << f->GetLocalPath() << " is in shutdown, ref_cnt = " << cnt << " -- waiting"); } @@ -611,7 +607,7 @@ void Cache::dec_ref_cnt(File* f, bool high_debug) } } - TRACE_INT(tlvl, "Cache::dec_ref_cnt " << f->GetLocalPath() << ", cnt at entry = " << cnt); + TRACE_INT(tlvl, "dec_ref_cnt " << f->GetLocalPath() << ", cnt at entry = " << cnt); if (cnt == 1) { @@ -620,54 +616,52 @@ void Cache::dec_ref_cnt(File* f, bool high_debug) // Note, here we "reuse" the existing reference count for the // final sync. - TRACE(Debug, "Cache::dec_ref_cnt " << f->GetLocalPath() << ", scheduling final sync"); + TRACE(Debug, "dec_ref_cnt " << f->GetLocalPath() << ", scheduling final sync"); schedule_file_sync(f, true, true); return; } } { - XrdSysCondVarHelper lock(&m_active_cond); + XrdSysCondVarHelper lock(&m_active_cond); - cnt = f->dec_ref_cnt(); - TRACE_INT(tlvl, "Cache::dec_ref_cnt " << f->GetLocalPath() << ", cnt after sync_check and dec_ref_cnt = " << cnt); - if (cnt == 0) - { - ActiveMap_i it = m_active.find(f->GetLocalPath()); - m_active.erase(it); + cnt = f->dec_ref_cnt(); + TRACE_INT(tlvl, "dec_ref_cnt " << f->GetLocalPath() << ", cnt after sync_check and dec_ref_cnt = " << cnt); + if (cnt == 0) + { + ActiveMap_i it = m_active.find(f->GetLocalPath()); + m_active.erase(it); - if (m_configuration.are_dirstats_enabled()) - { - m_closed_files_stats.insert(std::make_pair(f->GetLocalPath(), f->DeltaStatsFromLastCall())); - } + m_closed_files_stats.insert(std::make_pair(f->GetLocalPath(), f->DeltaStatsFromLastCall())); - if (m_gstream) - { - const Info::AStat *as = f->GetLastAccessStats(); - - char buf[4096]; - int len = snprintf(buf, 4096, "{\"event\":\"file_close\"," - "\"lfn\":\"%s\",\"size\":%lld,\"blk_size\":%d,\"n_blks\":%d,\"n_blks_done\":%d," - "\"access_cnt\":%lu,\"attach_t\":%lld,\"detach_t\":%lld," - "\"b_hit\":%lld,\"b_miss\":%lld,\"b_bypass\":%lld}", - f->GetLocalPath().c_str(), f->GetFileSize(), f->GetBlockSize(), - f->GetNBlocks(), f->GetNDownloadedBlocks(), - (unsigned long) f->GetAccessCnt(), (long long) as->AttachTime, (long long) as->DetachTime, - as->BytesHit, as->BytesMissed, as->BytesBypassed - ); - bool suc = false; - if (len < 4096) - { - suc = m_gstream->Insert(buf, len + 1); - } - if ( ! suc) - { - TRACE(Error, "Failed g-stream insertion of file_close record."); - } - } + if (m_gstream) + { + const Info::AStat *as = f->GetLastAccessStats(); + + char buf[4096]; + int len = snprintf(buf, 4096, "{\"event\":\"file_close\"," + "\"lfn\":\"%s\",\"size\":%lld,\"blk_size\":%d,\"n_blks\":%d,\"n_blks_done\":%d," + "\"access_cnt\":%lu,\"attach_t\":%lld,\"detach_t\":%lld,\"remotes\":%s," + "\"b_hit\":%lld,\"b_miss\":%lld,\"b_bypass\":%lld}", + f->GetLocalPath().c_str(), f->GetFileSize(), f->GetBlockSize(), + f->GetNBlocks(), f->GetNDownloadedBlocks(), + (unsigned long) f->GetAccessCnt(), (long long) as->AttachTime, (long long) as->DetachTime, + f->GetRemoteLocations().c_str(), + as->BytesHit, as->BytesMissed, as->BytesBypassed + ); + bool suc = false; + if (len < 4096) + { + suc = m_gstream->Insert(buf, len + 1); + } + if ( ! suc) + { + TRACE(Error, "Failed g-stream insertion of file_close record, len=" << len); + } + } - delete f; - } + delete f; + } } } @@ -790,7 +784,7 @@ int Cache::LocalFilePath(const char *curl, char *buff, int blen, static const mode_t worldReadable = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH; static const char *lfpReason[] = { "ForAccess", "ForInfo", "ForPath" }; - TRACE(Debug, "Cache::LocalFilePath '" << curl << "', why=" << lfpReason[why]); + TRACE(Debug, "LocalFilePath '" << curl << "', why=" << lfpReason[why]); if (buff && blen > 0) buff[0] = 0; @@ -801,7 +795,7 @@ int Cache::LocalFilePath(const char *curl, char *buff, int blen, if (why == ForPath) { int ret = m_oss->Lfn2Pfn(f_name.c_str(), buff, blen); - TRACE(Info, "Cache::LocalFilePath '" << curl << "', why=" << lfpReason[why] << " -> " << ret); + TRACE(Info, "LocalFilePath '" << curl << "', why=" << lfpReason[why] << " -> " << ret); return ret; } @@ -814,9 +808,9 @@ int Cache::LocalFilePath(const char *curl, char *buff, int blen, if (m_oss->Stat(f_name.c_str(), &sbuff) == XrdOssOK && m_oss->Stat(i_name.c_str(), &sbuff2) == XrdOssOK) { - if ( S_ISDIR(sbuff.st_mode)) + if (S_ISDIR(sbuff.st_mode)) { - TRACE(Info, "Cache::LocalFilePath '" << curl << "', why=" << lfpReason[why] << " -> EISDIR"); + TRACE(Info, "LocalFilePath '" << curl << "', why=" << lfpReason[why] << " -> EISDIR"); return -EISDIR; } else @@ -824,11 +818,16 @@ int Cache::LocalFilePath(const char *curl, char *buff, int blen, bool read_ok = false; bool is_complete = false; - // Lock and check if file is active. If NOT, keep the lock + // Lock and check if the file is active. If NOT, keep the lock // and add dummy access after successful reading of info file. // If it IS active, just release the lock, this ongoing access will // assure the file continues to exist. + // XXXX How can I just loop over the cinfo file when active? + // Can I not get is_complete from the existing file? + // Do I still want to inject access record? + // Oh, it writes only if not active .... still let's try to use existing File. + m_active_cond.Lock(); bool is_active = m_active.find(f_name) != m_active.end(); @@ -841,7 +840,7 @@ int Cache::LocalFilePath(const char *curl, char *buff, int blen, if (res >= 0) { Info info(m_trace, 0); - if (info.Read(infoFile, i_name)) + if (info.Read(infoFile, i_name.c_str())) { read_ok = true; @@ -851,7 +850,7 @@ int Cache::LocalFilePath(const char *curl, char *buff, int blen, if ( ! is_active && is_complete && why == ForAccess) { info.WriteIOStatSingle(info.GetFileSize()); - info.Write(infoFile); + info.Write(infoFile, i_name.c_str()); } } infoFile->Close(); @@ -873,14 +872,14 @@ int Cache::LocalFilePath(const char *curl, char *buff, int blen, if (why == ForAccess) {mode_t mode = (forall ? worldReadable : groupReadable); if (((sbuff.st_mode & worldReadable) != mode) - && (m_oss->Chmod(f_name.c_str(),mode) != XrdOssOK)) + && (m_oss->Chmod(f_name.c_str(), mode) != XrdOssOK)) {is_complete = false; *buff = 0; } } } - TRACE(Info, "Cache::LocalFilePath '" << curl << "', why=" << lfpReason[why] << + TRACE(Info, "LocalFilePath '" << curl << "', why=" << lfpReason[why] << (is_complete ? " -> FILE_COMPLETE_IN_CACHE" : " -> EREMOTE")); return is_complete ? 0 : -EREMOTE; @@ -888,7 +887,7 @@ int Cache::LocalFilePath(const char *curl, char *buff, int blen, } } - TRACE(Info, "Cache::LocalFilePath '" << curl << "', why=" << lfpReason[why] << " -> ENOENT"); + TRACE(Info, "LocalFilePath '" << curl << "', why=" << lfpReason[why] << " -> ENOENT"); return -ENOENT; } @@ -912,7 +911,7 @@ int Cache::Prepare(const char *curl, int oflags, mode_t mode) // Do not allow write access. if (oflags & (O_WRONLY | O_RDWR | O_APPEND | O_CREAT)) { - TRACE(Warning, "Cache::Prepare write access requested on file " << f_name << ". Denying access."); + TRACE(Warning, "Prepare write access requested on file " << f_name << ". Denying access."); return -EROFS; } @@ -938,7 +937,7 @@ int Cache::Prepare(const char *curl, int oflags, mode_t mode) int res = m_oss->Stat(i_name.c_str(), &sbuff); if (res == 0) { - TRACE(Dump, "Cache::Prepare defer open " << f_name); + TRACE(Dump, "Prepare defer open " << f_name); return 1; } else @@ -959,7 +958,6 @@ int Cache::Stat(const char *curl, struct stat &sbuff) { XrdCl::URL url(curl); std::string f_name = url.GetPath(); - std::string i_name = f_name + Info::s_infoExtension; { XrdSysCondVarHelper lock(&m_active_cond); @@ -977,11 +975,13 @@ int Cache::Stat(const char *curl, struct stat &sbuff) bool success = false; XrdOssDF* infoFile = m_oss->newFile(m_configuration.m_username.c_str()); XrdOucEnv myEnv; - int res = infoFile->Open(i_name.c_str(), O_RDONLY, 0600, myEnv); + + f_name += Info::s_infoExtension; + int res = infoFile->Open(f_name.c_str(), O_RDONLY, 0600, myEnv); if (res >= 0) { Info info(m_trace, 0); - if (info.Read(infoFile, i_name)) + if (info.Read(infoFile, f_name.c_str())) { sbuff.st_size = info.GetFileSize(); success = true; @@ -1008,7 +1008,7 @@ int Cache::Unlink(const char *curl) XrdCl::URL url(curl); std::string f_name = url.GetPath(); - // printf("Cache::Unlink url=%s\n\t fname=%s\n", curl, f_name.c_str()); + // printf("Unlink url=%s\n\t fname=%s\n", curl, f_name.c_str()); return UnlinkCommon(f_name, false); } @@ -1032,7 +1032,7 @@ int Cache::UnlinkCommon(const std::string& f_name, bool fail_if_open) { if (fail_if_open) { - TRACE(Info, "Cache::UnlinkCommon " << f_name << ", file currently open and force not requested - denying request"); + TRACE(Info, "UnlinkCommon " << f_name << ", file currently open and force not requested - denying request"); return -EBUSY; } @@ -1040,7 +1040,7 @@ int Cache::UnlinkCommon(const std::string& f_name, bool fail_if_open) // Attach() with possible File::Open(). Ask for retry. if (it->second == 0) { - TRACE(Info, "Cache::UnlinkCommon " << f_name << ", an operation on this file is ongoing - denying request"); + TRACE(Info, "UnlinkCommon " << f_name << ", an operation on this file is ongoing - denying request"); return -EAGAIN; } @@ -1065,7 +1065,7 @@ int Cache::UnlinkCommon(const std::string& f_name, bool fail_if_open) int f_ret = m_oss->Unlink(f_name.c_str()); int i_ret = m_oss->Unlink(i_name.c_str()); - TRACE(Debug, "Cache::UnlinkCommon " << f_name << ", f_ret=" << f_ret << ", i_ret=" << i_ret); + TRACE(Debug, "UnlinkCommon " << f_name << ", f_ret=" << f_ret << ", i_ret=" << i_ret); { XrdSysCondVarHelper lock(&m_active_cond); diff --git a/src/XrdPfc/XrdPfc.hh b/src/XrdPfc/XrdPfc.hh index 62d36ea9d5e..757ad7dd8b7 100644 --- a/src/XrdPfc/XrdPfc.hh +++ b/src/XrdPfc/XrdPfc.hh @@ -37,11 +37,6 @@ class XrdSysError; class XrdSysTrace; class XrdXrootdGStream; -namespace XrdCl -{ -class Log; -} - namespace XrdPfc { class File; @@ -59,42 +54,26 @@ namespace XrdPfc //---------------------------------------------------------------------------- struct Configuration { - Configuration() : - m_hdfsmode(false), - m_allow_xrdpfc_command(false), - m_data_space("public"), - m_meta_space("public"), - m_diskTotalSpace(-1), - m_diskUsageLWM(-1), - m_diskUsageHWM(-1), - m_fileUsageBaseline(-1), - m_fileUsageNominal(-1), - m_fileUsageMax(-1), - m_purgeInterval(300), - m_purgeColdFilesAge(-1), - m_purgeColdFilesPeriod(-1), - m_accHistorySize(20), - m_dirStatsMaxDepth(-1), - m_dirStatsStoreDepth(-1), - m_dirStats(false), - m_bufferSize(1024*1024), - m_RamAbsAvailable(0), - m_RamKeepStdBlocks(0), - m_wqueue_blocks(16), - m_wqueue_threads(4), - m_prefetch_max_blocks(10), - m_hdfsbsize(128*1024*1024), - m_flushCnt(2000) - {} + Configuration(); bool are_file_usage_limits_set() const { return m_fileUsageMax > 0; } - bool is_age_based_purge_in_effect() const { return m_purgeColdFilesAge > 0; } + bool is_age_based_purge_in_effect() const { return m_purgeColdFilesAge > 0 ; } + bool is_uvkeep_purge_in_effect() const { return m_cs_UVKeep >= 0; } + bool is_dir_stat_reporting_on() const { return m_dirStatsMaxDepth >= 0 || ! m_dirStatsDirs.empty() || ! m_dirStatsDirGlobs.empty(); } bool is_purge_plugin_set_up() const { return false; } void calculate_fractional_usages(long long du, long long fu, double &frac_du, double &frac_fu); - // This might become more complicated with per-dir purge policy - bool are_dirstats_enabled() const { return m_dirStats; } + CkSumCheck_e get_cs_Chk() const { return (CkSumCheck_e) m_cs_Chk; } + + bool is_cschk_cache() const { return m_cs_Chk & CSChk_Cache; } + bool is_cschk_net() const { return m_cs_Chk & CSChk_Net; } + bool is_cschk_any() const { return m_cs_Chk & CSChk_Both; } + bool is_cschk_both() const { return (m_cs_Chk & CSChk_Both) == CSChk_Both; } + + bool does_cschk_have_missing_bits(CkSumCheck_e cks_on_file) const { return m_cs_Chk & ~cks_on_file; } + + bool should_uvkeep_purge(time_t delta) const { return m_cs_UVKeep >= 0 && delta > m_cs_UVKeep; } bool m_hdfsmode; //!< flag for enabling block-level operation bool m_allow_xrdpfc_command; //!< flag for enabling access to /xrdpfc-command/ functionality. @@ -111,14 +90,13 @@ struct Configuration long long m_fileUsageMax; //!< cache purge - files usage maximum int m_purgeInterval; //!< sleep interval between cache purges int m_purgeColdFilesAge; //!< purge files older than this age - int m_purgeColdFilesPeriod; //!< peform cold file purge every this many purge cycles + int m_purgeAgeBasedPeriod; //!< peform cold file / uvkeep purge every this many purge cycles int m_accHistorySize; //!< max number of entries in access history part of cinfo file std::set m_dirStatsDirs; //!< directories for which stat reporting was requested std::set m_dirStatsDirGlobs; //!< directory globs for which stat reporting was requested int m_dirStatsMaxDepth; //!< maximum depth for statistics write out int m_dirStatsStoreDepth; //!< depth to which statistics should be collected - bool m_dirStats; //!< is directory access / usage statistics enabled long long m_bufferSize; //!< prefetch buffer size, default 1MB long long m_RamAbsAvailable; //!< available from configuration @@ -129,6 +107,10 @@ struct Configuration long long m_hdfsbsize; //!< used with m_hdfsmode, default 128MB long long m_flushCnt; //!< nuber of unsynced blcoks on disk before flush is called + + time_t m_cs_UVKeep; //!< unverified checksum cache keep + int m_cs_Chk; //!< Checksum check + bool m_cs_ChkTLS; //!< Allow TLS }; //------------------------------------------------------------------------------ @@ -340,7 +322,9 @@ public: //--------------------------------------------------------------------- //! Singleton access. //--------------------------------------------------------------------- - static Cache &GetInstance(); + static Cache &GetInstance(); + static const Cache &TheOne(); + static const Configuration &Conf(); //--------------------------------------------------------------------- //! Version check. @@ -427,8 +411,7 @@ private: XrdSysTrace *m_trace; const char *m_traceID; - XrdOucCacheStats m_ouc_stats; //!< - XrdOss *m_oss; //!< disk cache file system + XrdOss *m_oss; //!< disk cache file system XrdXrootdGStream *m_gstream; @@ -445,13 +428,7 @@ private: std::list m_RAM_std_blocks; //!< A list of blocks of standard size, to be reused. int m_RAM_std_size; - int m_csUVKeep; //!< unverified checksum cache keep bool m_isClient; //!< True if running as client - char m_csChk; //!< Checksum check - static const int csChk_None = 0; - static const int csChk_Cache = 1; - static const int csChk_Net = 2; - static const int csChk_TLS = 4; struct WriteQ { diff --git a/src/XrdPfc/XrdPfcCommand.cc b/src/XrdPfc/XrdPfcCommand.cc index ea811509872..fc8f0dc1e1f 100644 --- a/src/XrdPfc/XrdPfcCommand.cc +++ b/src/XrdPfc/XrdPfcCommand.cc @@ -244,7 +244,7 @@ void Cache::ExecuteCommandUrl(const std::string& command_url) myInfo.WriteIOStatSingle(file_size, att_time, att_time + access_duration[i]); } - myInfo.Write(myInfoFile); + myInfo.Write(myInfoFile, cinfo_path.c_str()); myInfoFile->Close(); delete myInfoFile; myFile->Close(); delete myFile; diff --git a/src/XrdPfc/XrdPfcConfiguration.cc b/src/XrdPfc/XrdPfcConfiguration.cc index 299abd1838a..46ea7a58044 100644 --- a/src/XrdPfc/XrdPfcConfiguration.cc +++ b/src/XrdPfc/XrdPfcConfiguration.cc @@ -19,10 +19,41 @@ using namespace XrdPfc; XrdVERSIONINFO(XrdOucGetCache, XrdPfc); +Configuration::Configuration() : + m_hdfsmode(false), + m_allow_xrdpfc_command(false), + m_data_space("public"), + m_meta_space("public"), + m_diskTotalSpace(-1), + m_diskUsageLWM(-1), + m_diskUsageHWM(-1), + m_fileUsageBaseline(-1), + m_fileUsageNominal(-1), + m_fileUsageMax(-1), + m_purgeInterval(300), + m_purgeColdFilesAge(-1), + m_purgeAgeBasedPeriod(10), + m_accHistorySize(20), + m_dirStatsMaxDepth(-1), + m_dirStatsStoreDepth(0), + m_bufferSize(1024*1024), + m_RamAbsAvailable(0), + m_RamKeepStdBlocks(0), + m_wqueue_blocks(16), + m_wqueue_threads(4), + m_prefetch_max_blocks(10), + m_hdfsbsize(128*1024*1024), + m_flushCnt(2000), + m_cs_UVKeep(-1), + m_cs_Chk(CSChk_Net), + m_cs_ChkTLS(false) +{} + + bool Cache::cfg2bytes(const std::string &str, long long &store, long long totalSpace, const char *name) { char errStr[1024]; - snprintf(errStr, 1024, "Cache::ConfigParameters() Error parsing parameter %s", name); + snprintf(errStr, 1024, "ConfigParameters() Error parsing parameter %s", name); if (::isalpha(*(str.rbegin()))) { @@ -47,7 +78,7 @@ bool Cache::cfg2bytes(const std::string &str, long long &store, long long totalS if (store < 0 || store > totalSpace) { - snprintf(errStr, 1024, "Cache::ConfigParameters() Error: parameter %s should be between 0 and total available disk space (%lld) - it is %lld (given as %s)", + snprintf(errStr, 1024, "ConfigParameters() Error: parameter %s should be between 0 and total available disk space (%lld) - it is %lld (given as %s)", name, totalSpace, store, str.c_str()); m_log.Emsg(errStr, ""); return false; @@ -80,10 +111,10 @@ bool Cache::xcschk(XrdOucStream &Config) const char *val, *val2; struct cschkopts {const char *opname; int opval;} csopts[] = { - {"off", csChk_None}, - {"cache", csChk_Cache}, - {"net", csChk_Net}, - {"tls", csChk_TLS} + {"off", CSChk_None}, + {"cache", CSChk_Cache}, + {"net", CSChk_Net}, + {"tls", CSChk_TLS} }; int i, numopts = sizeof(csopts)/sizeof(struct cschkopts); bool isNo; @@ -92,36 +123,53 @@ bool Cache::xcschk(XrdOucStream &Config) {m_log.Emsg("Config", "cschk parameter not specified"); return false; } while(val) - {if ((isNo = strncmp(val, "no", 2) == 0)) val2 = val+2; - else val2 = val; - for (i = 0; i < numopts; i++) - { - if (! strcmp(val2, csopts[i].opname)) - { - if (isNo) m_csChk &= ~csopts[i].opval; - else if (csopts[i].opval) m_csChk |= csopts[i].opval; - else m_csChk = csopts[i].opval; - break; - } - } - if (i >= numopts) - { if (strcmp(val, "uvkeep")) - { m_log.Emsg("Config", "invalid cschk option -", val); - return false; - } - if (! (val = Config.GetWord())) - { m_log.Emsg("Config","cschk uvkeep value not specified"); - return false; - } - if (!strcmp(val, "lru")) m_csUVKeep = -1; - else if (XrdOuca2x::a2tm(m_log,"uvkeep time",val,&m_csUVKeep,0)) - return false; - } - val = Config.GetWord(); - } + { + if ((isNo = strncmp(val, "no", 2) == 0)) + val2 = val + 2; + else + val2 = val; + for (i = 0; i < numopts; i++) + { + if (!strcmp(val2, csopts[i].opname)) + { + if (isNo) + m_configuration.m_cs_Chk &= ~csopts[i].opval; + else if (csopts[i].opval) + m_configuration.m_cs_Chk |= csopts[i].opval; + else + m_configuration.m_cs_Chk = csopts[i].opval; + break; + } + } + if (i >= numopts) + { + if (strcmp(val, "uvkeep")) + { + m_log.Emsg("Config", "invalid cschk option -", val); + return false; + } + if (!(val = Config.GetWord())) + { + m_log.Emsg("Config", "cschk uvkeep value not specified"); + return false; + } + if (!strcmp(val, "lru")) + m_configuration.m_cs_UVKeep = -1; + else + { + int uvkeep; + if (XrdOuca2x::a2tm(m_log, "uvkeep time", val, &uvkeep, 0)) + return false; + m_configuration.m_cs_UVKeep = uvkeep; + } + } + val = Config.GetWord(); + } + // Decompose into separate TLS state, it is only passed on to psx + m_configuration.m_cs_ChkTLS = m_configuration.m_cs_Chk & CSChk_TLS; + m_configuration.m_cs_Chk &= ~CSChk_TLS; - if (!(m_csChk & csChk_Net)) m_env->Put("psx.CSNet", "0"); - else m_env->Put("psx.CSNet", (m_csChk & csChk_TLS ? "2" : "1")); + m_env->Put("psx.CSNet", m_configuration.is_cschk_net() ? (m_configuration.m_cs_ChkTLS ? "2" : "1") : "0"); return true; } @@ -168,7 +216,7 @@ bool Cache::xdlib(XrdOucStream &Config) Decision * d = ep(m_log); if (! d) { - TRACE(Error, "Cache::Config() decisionlib was not able to create a decision object"); + TRACE(Error, "Config() decisionlib was not able to create a decision object"); return false; } if (params[0]) @@ -230,14 +278,14 @@ bool Cache::Config(const char *config_filename, const char *parameters) if (! config_filename || ! *config_filename) { - TRACE(Error, "Cache::Config() configuration file not specified."); + TRACE(Error, "Config() configuration file not specified."); return false; } int fd; if ( (fd = open(config_filename, O_RDONLY, 0)) < 0) { - TRACE( Error, "Cache::Config() can't open configuration file " << config_filename); + TRACE( Error, "Config() can't open configuration file " << config_filename); return false; } @@ -261,9 +309,7 @@ bool Cache::Config(const char *config_filename, const char *parameters) } // If network checksum processing is the default, indicate so. - // - if (m_csChk & csChk_Net) - m_env->Put("psx.CSNet", (m_csChk & csChk_TLS ? "2" : "1")); + if (m_configuration.is_cschk_net()) m_env->Put("psx.CSNet", m_configuration.m_cs_ChkTLS ? "2" : "1"); // Actual parsing of the config file. bool retval = true, aOK = true; @@ -297,7 +343,7 @@ bool Cache::Config(const char *config_filename, const char *parameters) if ( ! retval) { - TRACE(Error, "Cache::Config() error in parsing"); + TRACE(Error, "Config() error in parsing"); aOK = false; } } @@ -306,14 +352,24 @@ bool Cache::Config(const char *config_filename, const char *parameters) // Load OSS plugin. myEnv.Put("oss.runmode", "pfc"); + if (m_configuration.is_cschk_cache()) + { + char csi_conf[128]; + if (snprintf(csi_conf, 128, "space=%s nofill", m_configuration.m_meta_space.c_str()) < 128) + { + ofsCfg->Push(XrdOfsConfigPI::theOssLib, "libXrdOssCsi", csi_conf); + } else { + TRACE(Error, "Config() buffer too small for libXrdOssCsi params."); + return false; + } + } if (ofsCfg->Load(XrdOfsConfigPI::theOssLib, &myEnv)) { ofsCfg->Plugin(m_oss); } else { - TRACE(Error, "Cache::Config() Unable to create an OSS object"); - m_oss = 0; + TRACE(Error, "Config() Unable to create an OSS object"); return false; } @@ -322,12 +378,12 @@ bool Cache::Config(const char *config_filename, const char *parameters) { if (m_oss->StatVS(&sP, m_configuration.m_data_space.c_str(), 1) < 0) { - m_log.Emsg("Cache::ConfigParameters()", "error obtaining stat info for data space ", m_configuration.m_data_space.c_str()); + m_log.Emsg("ConfigParameters()", "error obtaining stat info for data space ", m_configuration.m_data_space.c_str()); return false; } if (sP.Total < 10ll << 20) { - m_log.Emsg("Cache::ConfigParameters()", "available data space is less than 10 MB (can be due to a mistake in oss.localroot directive) for space ", + m_log.Emsg("ConfigParameters()", "available data space is less than 10 MB (can be due to a mistake in oss.localroot directive) for space ", m_configuration.m_data_space.c_str()); return false; } @@ -339,7 +395,7 @@ bool Cache::Config(const char *config_filename, const char *parameters) { if (m_configuration.m_diskUsageLWM >= m_configuration.m_diskUsageHWM) { printf("GGGG %lld %lld\n", m_configuration.m_diskUsageLWM, m_configuration.m_diskUsageHWM); - m_log.Emsg("Cache::ConfigParameters()", "pfc.diskusage should have lowWatermark < highWatermark."); + m_log.Emsg("ConfigParameters()", "pfc.diskusage should have lowWatermark < highWatermark."); aOK = false; } } @@ -355,7 +411,7 @@ bool Cache::Config(const char *config_filename, const char *parameters) m_configuration.m_fileUsageBaseline >= m_configuration.m_fileUsageMax || m_configuration.m_fileUsageNominal >= m_configuration.m_fileUsageMax) { - m_log.Emsg("Cache::ConfigParameters()", "pfc.diskusage files should have baseline < nominal < max."); + m_log.Emsg("ConfigParameters()", "pfc.diskusage files should have baseline < nominal < max."); aOK = false; } } @@ -413,9 +469,11 @@ bool Cache::Config(const char *config_filename, const char *parameters) // 111 "cache net tls"}; char buff[8192], uvk[32]; - if (m_csUVKeep < 0) strcpy(uvk, "lru"); - else sprintf(uvk, "%d", m_csUVKeep); - float rg = (m_configuration.m_RamAbsAvailable) / float(1024*1024*1024); + if (m_configuration.m_cs_UVKeep < 0) + strcpy(uvk, "lru"); + else + sprintf(uvk, "%ld", m_configuration.m_cs_UVKeep); + float rg = (m_configuration.m_RamAbsAvailable) / float(1024*1024*1024); loff = snprintf(buff, sizeof(buff), "Config effective %s pfc configuration:\n" " pfc.cschk %s uvkeep %s\n" " pfc.blocksize %lld\n" @@ -429,7 +487,7 @@ bool Cache::Config(const char *config_filename, const char *parameters) " pfc.flush %lld\n" " pfc.acchistorysize %d\n", config_filename, - csc[int(m_csChk)], uvk, + csc[int(m_configuration.m_cs_Chk)], uvk, m_configuration.m_bufferSize, m_configuration.m_prefetch_max_blocks, rg, @@ -444,7 +502,7 @@ bool Cache::Config(const char *config_filename, const char *parameters) m_configuration.m_flushCnt, m_configuration.m_accHistorySize); - if (m_configuration.are_dirstats_enabled()) + if (m_configuration.is_dir_stat_reporting_on()) { loff += snprintf(buff + loff, sizeof(buff) - loff, " pfc.dirstats maxdepth %d ((internal: store_depth %d, size_of_dirlist %d, size_of_globlist %d))\n", @@ -489,6 +547,21 @@ bool Cache::Config(const char *config_filename, const char *parameters) if (ofsCfg) delete ofsCfg; + // XXXX-CKSUM Testing. To be removed after OssPgi is also merged and valildated. + // Building of xrdpfc_print fails when this is enabled. +#ifdef XRDPFC_CKSUM_TEST + { + int xxx = m_configuration.m_cs_Chk; + + for (m_configuration.m_cs_Chk = CSChk_None; m_configuration.m_cs_Chk <= CSChk_Both; ++m_configuration.m_cs_Chk) + { + Info::TestCksumStuff(); + } + + m_configuration.m_cs_Chk = xxx; + } +#endif + return aOK; } @@ -560,7 +633,7 @@ bool Cache::ConfigParameters(std::string part, XrdOucStream& config, TmpConfigur { return false; } - if (XrdOuca2x::a2i(m_log, "Error getting purgecoldfiles period", cwg.GetWord(), &m_configuration.m_purgeColdFilesPeriod, 1, 1000)) + if (XrdOuca2x::a2i(m_log, "Error getting purgecoldfiles period", cwg.GetWord(), &m_configuration.m_purgeAgeBasedPeriod, 1, 1000)) { return false; } @@ -580,14 +653,12 @@ bool Cache::ConfigParameters(std::string part, XrdOucStream& config, TmpConfigur } else if ( part == "dirstats" ) { - m_configuration.m_dirStats = true; - const char *p = 0; - while ((p = cwg.GetWord())) + while ((p = cwg.GetWord()) && cwg.HasLast()) { if (strcmp(p, "maxdepth") == 0) { - if (XrdOuca2x::a2i(m_log, "Error getting maxdepth value", cwg.GetWord(), &m_configuration.m_dirStatsMaxDepth, 0, 5)) + if (XrdOuca2x::a2i(m_log, "Error getting maxdepth value", cwg.GetWord(), &m_configuration.m_dirStatsMaxDepth, 0, 16)) { return false; } @@ -646,7 +717,8 @@ bool Cache::ConfigParameters(std::string part, XrdOucStream& config, TmpConfigur } else { - m_log.Emsg("Config", "Error: dirstats stanza contains unknown directive", p); + m_log.Emsg("Config", "Error: dirstats stanza contains unknown directive '", p, "'"); + return false; } } } @@ -658,6 +730,12 @@ bool Cache::ConfigParameters(std::string part, XrdOucStream& config, TmpConfigur { return false; } + if (m_configuration.m_bufferSize & 0xFFF) + { + m_configuration.m_bufferSize &= ~0x0FFF; + m_configuration.m_bufferSize += 0x1000; + m_log.Emsg("Config", "pfc.blocksize must be a multiple of 4 kB. Rounded up."); + } } else if ( part == "prefetch" || part == "nramprefetch" ) { @@ -745,7 +823,7 @@ bool Cache::ConfigParameters(std::string part, XrdOucStream& config, TmpConfigur } else { - m_log.Emsg("Cache::ConfigParameters() unmatched pfc parameter", part.c_str()); + m_log.Emsg("ConfigParameters() unmatched pfc parameter", part.c_str()); return false; } diff --git a/src/XrdPfc/XrdPfcFile.cc b/src/XrdPfc/XrdPfcFile.cc index 42b92d441c1..ad1f79f53a7 100644 --- a/src/XrdPfc/XrdPfcFile.cc +++ b/src/XrdPfc/XrdPfcFile.cc @@ -77,7 +77,7 @@ File::~File() { if (m_info_file) { - TRACEF(Debug, "File::~File() close info "); + TRACEF(Debug, "~File() close info "); m_info_file->Close(); delete m_info_file; m_info_file = NULL; @@ -85,13 +85,13 @@ File::~File() if (m_data_file) { - TRACEF(Debug, "File::~File() close output "); + TRACEF(Debug, "~File() close output "); m_data_file->Close(); delete m_data_file; m_data_file = NULL; } - TRACEF(Debug, "File::~File() ended, prefetch score = " << m_prefetch_score); + TRACEF(Debug, "~File() ended, prefetch score = " << m_prefetch_score); } //------------------------------------------------------------------------------ @@ -155,7 +155,7 @@ Stats File::DeltaStatsFromLastCall() void File::BlockRemovedFromWriteQ(Block* b) { - TRACEF(Dump, "File::BlockRemovedFromWriteQ() block = " << (void*) b << " idx= " << b->m_offset/m_cfi.GetBufferSize()); + TRACEF(Dump, "BlockRemovedFromWriteQ() block = " << (void*) b << " idx= " << b->m_offset/m_cfi.GetBufferSize()); XrdSysCondVarHelper _lck(m_state_cond); dec_ref_count(b); @@ -163,7 +163,7 @@ void File::BlockRemovedFromWriteQ(Block* b) void File::BlocksRemovedFromWriteQ(std::list& blocks) { - TRACEF(Dump, "File::BlocksRemovedFromWriteQ() n_blocks = " << blocks.size()); + TRACEF(Dump, "BlocksRemovedFromWriteQ() n_blocks = " << blocks.size()); XrdSysCondVarHelper _lck(m_state_cond); @@ -175,11 +175,22 @@ void File::BlocksRemovedFromWriteQ(std::list& blocks) //------------------------------------------------------------------------------ +void File::ioUpdated(IO *io) +{ + std::string loc(io->GetLocation()); + XrdSysCondVarHelper _lck(m_state_cond); + insert_remote_location(loc); +} + +//------------------------------------------------------------------------------ + bool File::ioActive(IO *io) { // Returns true if delay is needed. - TRACEF(Debug, "File::ioActive start for io " << io); + TRACEF(Debug, "ioActive start for io " << io); + + std::string loc(io->GetLocation()); { XrdSysCondVarHelper _lck(m_state_cond); @@ -203,6 +214,8 @@ bool File::ioActive(IO *io) "\tio_map.size() " << m_io_map.size() << ", block_map.size() " << m_block_map.size() << ", file"); + insert_remote_location(loc); + // XXX Intermediate check for 4.11 - 5.0 transition. // Can be removed for 5.1, including member IODetals::m_ioactive_false_reported. assert( ! mi->second.m_ioactive_false_reported && "ioActive already returned false"); @@ -272,11 +285,11 @@ bool File::FinalizeSyncBeforeExit() m_cfi.WriteIOStatDetach(loc_stats); m_detach_time_logged = true; m_in_sync = true; - TRACEF(Debug, "File::FinalizeSyncBeforeExit requesting sync to write detach stats"); + TRACEF(Debug, "FinalizeSyncBeforeExit requesting sync to write detach stats"); return true; } } - TRACEF(Debug, "File::FinalizeSyncBeforeExit sync not required"); + TRACEF(Debug, "FinalizeSyncBeforeExit sync not required"); return false; } @@ -286,9 +299,10 @@ void File::AddIO(IO *io) { // Called from Cache::GetFile() when a new IO asks for the file. - TRACEF(Debug, "File::AddIO() io = " << (void*)io); + TRACEF(Debug, "AddIO() io = " << (void*)io); - time_t now = time(0); + time_t now = time(0); + std::string loc(io->GetLocation()); m_state_cond.Lock(); @@ -299,6 +313,8 @@ void File::AddIO(IO *io) m_io_map.insert(std::make_pair(io, IODetails(now))); m_stats.IoAttach(); + insert_remote_location(loc); + if (m_prefetch_state == kStopped) { m_prefetch_state = kOn; @@ -307,7 +323,7 @@ void File::AddIO(IO *io) } else { - TRACEF(Error, "File::AddIO() io = " << (void*)io << " already registered."); + TRACEF(Error, "AddIO() io = " << (void*)io << " already registered."); } m_state_cond.UnLock(); @@ -319,7 +335,7 @@ void File::RemoveIO(IO *io) { // Called from Cache::ReleaseFile. - TRACEF(Debug, "File::RemoveIO() io = " << (void*)io); + TRACEF(Debug, "RemoveIO() io = " << (void*)io); time_t now = time(0); @@ -340,14 +356,14 @@ void File::RemoveIO(IO *io) if (m_io_map.empty() && m_prefetch_state != kStopped && m_prefetch_state != kComplete) { - TRACEF(Error, "File::RemoveIO() io = " << (void*)io << " Prefetching is not stopped/complete -- it should be by now."); + TRACEF(Error, "RemoveIO() io = " << (void*)io << " Prefetching is not stopped/complete -- it should be by now."); m_prefetch_state = kStopped; cache()->DeRegisterPrefetchFile(this); } } else { - TRACEF(Error, "File::RemoveIO() io = " << (void*)io << " is NOT registered."); + TRACEF(Error, "RemoveIO() io = " << (void*)io << " is NOT registered."); } m_state_cond.UnLock(); @@ -359,11 +375,13 @@ bool File::Open() { // Sets errno accordingly. - TRACEF(Dump, "File::Open() open file for disk cache"); + static const char *tpfx = "Open() "; + + TRACEF(Dump, tpfx << "open file for disk cache"); if (m_is_open) { - TRACEF(Error, "File::Open() file is already opened."); + TRACEF(Error, tpfx << "file is already opened."); return true; } @@ -388,7 +406,7 @@ bool File::Open() if ((res = myOss.Create(myUser, m_filename.c_str(), 0600, myEnv, XRDOSS_mkpath)) != XrdOssOK) { - TRACEF(Error, "File::Open() Create failed " << ERRNO_AND_ERRSTR(-res)); + TRACEF(Error, tpfx << "Create failed " << ERRNO_AND_ERRSTR(-res)); errno = -res; return false; } @@ -396,7 +414,7 @@ bool File::Open() m_data_file = myOss.newFile(myUser); if ((res = m_data_file->Open(m_filename.c_str(), O_RDWR, 0600, myEnv)) != XrdOssOK) { - TRACEF(Error, "File::Open() Open failed " << ERRNO_AND_ERRSTR(-res)); + TRACEF(Error, tpfx << "Open failed " << ERRNO_AND_ERRSTR(-res)); errno = -res; delete m_data_file; m_data_file = 0; return false; @@ -406,7 +424,7 @@ bool File::Open() myEnv.Put("oss.cgroup", conf.m_meta_space.c_str()); if ((res = myOss.Create(myUser, ifn.c_str(), 0600, myEnv, XRDOSS_mkpath)) != XrdOssOK) { - TRACE(Error, "File::Open() Create failed for info file " << ifn << ERRNO_AND_ERRSTR(-res)); + TRACE(Error, tpfx << "Create failed for info file " << ifn << ERRNO_AND_ERRSTR(-res)); errno = -res; m_data_file->Close(); delete m_data_file; m_data_file = 0; return false; @@ -415,7 +433,7 @@ bool File::Open() m_info_file = myOss.newFile(myUser); if ((res = m_info_file->Open(ifn.c_str(), O_RDWR, 0600, myEnv)) != XrdOssOK) { - TRACEF(Error, "File::Open() Open failed for info file " << ifn << ERRNO_AND_ERRSTR(-res)); + TRACEF(Error, tpfx << "Failed for info file " << ifn << ERRNO_AND_ERRSTR(-res)); errno = -res; delete m_info_file; m_info_file = 0; m_data_file->Close(); delete m_data_file; m_data_file = 0; @@ -424,9 +442,9 @@ bool File::Open() bool initialize_info_file = true; - if (info_existed && m_cfi.Read(m_info_file, ifn)) + if (info_existed && m_cfi.Read(m_info_file, ifn.c_str())) { - TRACEF(Debug, "Open - reading existing info file. (data_existed=" << data_existed << + TRACEF(Debug, tpfx << "Reading existing info file. (data_existed=" << data_existed << ", data_size_stat=" << (data_existed ? data_stat.st_size : -1ll) << ", data_size_from_last_block=" << m_cfi.GetExpectedDataFileSize() << ")"); @@ -434,21 +452,36 @@ bool File::Open() if (data_existed && data_stat.st_size >= m_cfi.GetExpectedDataFileSize()) { initialize_info_file = false; + } else { + TRACEF(Warning, tpfx << "Basic sanity checks on data file failed, resetting info file, truncating data file."); + m_cfi.ResetAllAccessStats(); + m_data_file->Ftruncate(0); } - else + } + + if ( ! initialize_info_file && m_cfi.GetCkSumState() != conf.get_cs_Chk()) + { + if (conf.does_cschk_have_missing_bits(m_cfi.GetCkSumState()) && + conf.should_uvkeep_purge(time(0) - m_cfi.GetNoCkSumTimeForUVKeep())) { - TRACEF(Warning, "Open - basic sanity checks on data file failed, resetting info file."); + TRACEF(Info, tpfx << "Cksum state of file insufficient, uvkeep test failed, resetting info file, truncating data file."); + initialize_info_file = true; m_cfi.ResetAllAccessStats(); + m_data_file->Ftruncate(0); + } else { + // If a file is complete, we don't really need to reset net cksums ... well, maybe next time. + m_cfi.DowngradeCkSumState(conf.get_cs_Chk()); } } + if (initialize_info_file) { m_cfi.SetBufferSize(conf.m_bufferSize); m_cfi.SetFileSize(m_file_size); - m_cfi.Write(m_info_file); + m_cfi.SetCkSumState(conf.get_cs_Chk()); + m_cfi.Write(m_info_file, ifn.c_str()); m_info_file->Fsync(); - int ss = (m_file_size - 1)/m_cfi.GetBufferSize() + 1; - TRACEF(Debug, "Creating new file info, data size = " << m_file_size << " num blocks = " << ss); + TRACEF(Debug, tpfx << "Creating new file info, data size = " << m_file_size << " num blocks = " << m_cfi.GetNBlocks()); } m_cfi.WriteIOStatAttach(); @@ -506,25 +539,33 @@ Block* File::PrepareBlockRequest(int i, IO *io, bool prefetch) // Reference count is 0 so increase it in calling function if you want to // catch the block while still in memory. - const long long BS = m_cfi.GetBufferSize(); - const int last_block = m_cfi.GetSizeInBits() - 1; - - long long off = i * BS; - long long this_bs = (i == last_block) ? m_file_size - off : BS; + const long long BS = m_cfi.GetBufferSize(); + const long long off = i * BS; + const int last_block = m_cfi.GetNBlocks() - 1; + const bool cs_net = cache()->RefConfiguration().is_cschk_net(); + + int blk_size, req_size; + if (i == last_block) { + blk_size = req_size = m_file_size - off; + if (cs_net && req_size & 0xFFF) req_size = (req_size & ~0xFFF) + 0x1000; + } else { + blk_size = req_size = BS; + } Block *b = 0; - char *buf = cache()->RequestRAM(this_bs); + char *buf = cache()->RequestRAM(req_size); if (buf) { - b = new (std::nothrow) Block(this, io, buf, off, this_bs, prefetch); + b = new (std::nothrow) Block(this, io, buf, off, blk_size, prefetch, cs_net); if (b) { m_block_map[i] = b; // Actual Read request is issued in ProcessBlockRequests(). - TRACEF(Dump, "File::PrepareBlockRequest() " << i << " prefetch " << prefetch << " address " << (void*) b); + TRACEF(Dump, "PrepareBlockRequest() idx=" << i << ", block=" << (void*) b << ", prefetch=" << prefetch << + ", offset=" << off << ", size=" << blk_size << ", buffer=" << (void*) buf); if (m_prefetch_state == kOn && (int) m_block_map.size() >= Cache::GetInstance().RefConfiguration().m_prefetch_max_blocks) { @@ -534,30 +575,35 @@ Block* File::PrepareBlockRequest(int i, IO *io, bool prefetch) } else { - TRACEF(Dump, "File::PrepareBlockRequest() " << i << " prefetch " << prefetch << ", allocation failed."); + TRACEF(Dump, "PrepareBlockRequest() " << i << " prefetch " << prefetch << ", allocation failed."); } } return b; } -void File::ProcessBlockRequest(Block *b, bool prefetch) +void File::ProcessBlockRequest(Block *b) { // This *must not* be called with block_map locked. - BlockResponseHandler* oucCB = new BlockResponseHandler(b, prefetch); - b->get_io()->GetInput()->Read(*oucCB, b->get_buff(), b->get_offset(), b->get_size()); + BlockResponseHandler* oucCB = new BlockResponseHandler(b); + if (b->req_cksum_net()) + { + int req_size = b->get_size(); + if (req_size & 0xFFF) req_size = (req_size & ~0xFFF) + 0x1000; + b->get_io()->GetInput()->pgRead(*oucCB, b->get_buff(), b->get_offset(), req_size, b->ref_cksum_vec()); + } else { + b->get_io()->GetInput()-> Read(*oucCB, b->get_buff(), b->get_offset(), b->get_size()); + } } -void File::ProcessBlockRequests(BlockList_t& blks, bool prefetch) +void File::ProcessBlockRequests(BlockList_t& blks) { // This *must not* be called with block_map locked. for (BlockList_i bi = blks.begin(); bi != blks.end(); ++bi) { - Block *b = *bi; - BlockResponseHandler* oucCB = new BlockResponseHandler(b, prefetch); - b->get_io()->GetInput()->Read(*oucCB, b->get_buff(), b->get_offset(), b->get_size()); + ProcessBlockRequest(*bi); } } @@ -595,7 +641,7 @@ int File::RequestBlocksDirect(IO *io, DirectResponseHandler *handler, IntList_t& int File::ReadBlocksFromDisk(std::list& blocks, char* req_buf, long long req_off, long long req_size) { - TRACEF(Dump, "File::ReadBlocksFromDisk " << blocks.size()); + TRACEF(Dump, "ReadBlocksFromDisk " << blocks.size()); const long long BS = m_cfi.GetBufferSize(); long long total = 0; @@ -612,17 +658,17 @@ int File::ReadBlocksFromDisk(std::list& blocks, overlap(*ii, BS, req_off, req_size, off, blk_off, size); long long rs = m_data_file->Read(req_buf + off, *ii * BS + blk_off -m_offset, size); - TRACEF(Dump, "File::ReadBlocksFromDisk block idx = " << *ii << " size= " << size); + TRACEF(Dump, "ReadBlocksFromDisk block idx = " << *ii << " size= " << size); if (rs < 0) { - TRACEF(Error, "File::ReadBlocksFromDisk neg retval = " << rs << " idx = " << *ii ); + TRACEF(Error, "ReadBlocksFromDisk neg retval = " << rs << " idx = " << *ii ); return rs; } if (rs != size) { - TRACEF(Error, "File::ReadBlocksFromDisk incomplete size = " << rs << " idx = " << *ii); + TRACEF(Error, "ReadBlocksFromDisk incomplete size = " << rs << " idx = " << *ii); return -EIO; } @@ -663,7 +709,7 @@ int File::Read(IO *io, char* iUserBuff, long long iUserOff, int iUserSize) if ( ! m_is_open) { m_state_cond.UnLock(); - TRACEF(Error, "File::Read file is not open"); + TRACEF(Error, "Read file is not open"); return io->GetInput()->Read(iUserBuff, iUserOff, iUserSize); } if (m_in_shutdown) @@ -674,20 +720,20 @@ int File::Read(IO *io, char* iUserBuff, long long iUserOff, int iUserSize) for (int block_idx = idx_first; block_idx <= idx_last; ++block_idx) { - TRACEF(Dump, "File::Read() idx " << block_idx); + TRACEF(Dump, "Read() idx " << block_idx); BlockMap_i bi = m_block_map.find(block_idx); // In RAM or incoming? if (bi != m_block_map.end()) { inc_ref_count(bi->second); - TRACEF(Dump, "File::Read() " << (void*) iUserBuff << "inc_ref_count for existing block " << bi->second << " idx = " << block_idx); + TRACEF(Dump, "Read() " << (void*) iUserBuff << "inc_ref_count for existing block " << bi->second << " idx = " << block_idx); blks_to_process.push_front(bi->second); } // On disk? else if (m_cfi.TestBitWritten(offsetIdx(block_idx))) { - TRACEF(Dump, "File::Read() read from disk " << (void*)iUserBuff << " idx = " << block_idx); + TRACEF(Dump, "Read() read from disk " << (void*)iUserBuff << " idx = " << block_idx); blks_on_disk.push_back(block_idx); } // Then we have to get it ... @@ -697,7 +743,7 @@ int File::Read(IO *io, char* iUserBuff, long long iUserOff, int iUserSize) Block *b = PrepareBlockRequest(block_idx, io, false); if (b) { - TRACEF(Dump, "File::Read() inc_ref_count new " << (void*)iUserBuff << " idx = " << block_idx); + TRACEF(Dump, "Read() inc_ref_count new " << (void*)iUserBuff << " idx = " << block_idx); inc_ref_count(b); blks_to_process.push_back(b); blks_to_request.push_back(b); @@ -706,7 +752,7 @@ int File::Read(IO *io, char* iUserBuff, long long iUserOff, int iUserSize) // Nope ... read this directly without caching. else { - TRACEF(Dump, "File::Read() direct block " << block_idx); + TRACEF(Dump, "Read() direct block " << block_idx); blks_direct.push_back(block_idx); } } @@ -714,7 +760,7 @@ int File::Read(IO *io, char* iUserBuff, long long iUserOff, int iUserSize) m_state_cond.UnLock(); - ProcessBlockRequests(blks_to_request, false); + ProcessBlockRequests(blks_to_request); long long bytes_read = 0; int error_cond = 0; // to be set to -errno @@ -730,14 +776,14 @@ int File::Read(IO *io, char* iUserBuff, long long iUserOff, int iUserSize) direct_size = RequestBlocksDirect(io, direct_handler, blks_direct, iUserBuff, iUserOff, iUserSize); - TRACEF(Dump, "File::Read() direct read requests sent out, size = " << direct_size); + TRACEF(Dump, "Read() direct read requests sent out, size = " << direct_size); } // Second, read blocks from disk. if ( ! blks_on_disk.empty() && bytes_read >= 0) { int rc = ReadBlocksFromDisk(blks_on_disk, iUserBuff, iUserOff, iUserSize); - TRACEF(Dump, "File::Read() " << (void*)iUserBuff <<" from disk finished size = " << rc); + TRACEF(Dump, "Read() " << (void*)iUserBuff <<" from disk finished size = " << rc); if (rc >= 0) { bytes_read += rc; @@ -746,7 +792,7 @@ int File::Read(IO *io, char* iUserBuff, long long iUserOff, int iUserSize) else { error_cond = rc; - TRACEF(Error, "File::Read() failed read from disk"); + TRACEF(Error, "Read() failed read from disk"); } } @@ -764,7 +810,7 @@ int File::Read(IO *io, char* iUserBuff, long long iUserOff, int iUserSize) { if ((*bi)->is_failed() && (*bi)->get_io() != io) { - TRACEF(Info, "File::Read() requested block " << (void*)(*bi) << " failed with another io " << + TRACEF(Info, "Read() requested block " << (void*)(*bi) << " failed with another io " << (*bi)->get_io() << " - reissuing request with my io " << io); (*bi)->reset_error_and_set_io(io); @@ -773,7 +819,7 @@ int File::Read(IO *io, char* iUserBuff, long long iUserOff, int iUserSize) } else if ((*bi)->is_finished()) { - TRACEF(Dump, "File::Read() requested block finished " << (void*)(*bi) << ", is_failed()=" << (*bi)->is_failed()); + TRACEF(Dump, "Read() requested block finished " << (void*)(*bi) << ", is_failed()=" << (*bi)->is_failed()); finished.push_back(*bi); BlockList_i bj = bi++; blks_to_process.erase(bj); @@ -791,7 +837,7 @@ int File::Read(IO *io, char* iUserBuff, long long iUserOff, int iUserSize) } } - ProcessBlockRequests(to_reissue, false); + ProcessBlockRequests(to_reissue); to_reissue.clear(); BlockList_i bi = finished.begin(); @@ -805,7 +851,7 @@ int File::Read(IO *io, char* iUserBuff, long long iUserOff, int iUserSize) overlap((*bi)->m_offset/BS, BS, iUserOff, iUserSize, user_off, off_in_block, size_to_copy); - TRACEF(Dump, "File::Read() ub=" << (void*)iUserBuff << " from finished block " << (*bi)->m_offset/BS << " size " << size_to_copy); + TRACEF(Dump, "Read() ub=" << (void*)iUserBuff << " from finished block " << (*bi)->m_offset/BS << " size " << size_to_copy); memcpy(&iUserBuff[user_off], &((*bi)->m_buff[off_in_block]), size_to_copy); bytes_read += size_to_copy; @@ -823,7 +869,7 @@ int File::Read(IO *io, char* iUserBuff, long long iUserOff, int iUserSize) if ( ! error_cond) { error_cond = (*bi)->m_errno; - TRACEF(Error, "File::Read() io " << io << ", block "<< (*bi)->m_offset/BS << + TRACEF(Error, "Read() io " << io << ", block "<< (*bi)->m_offset/BS << " finished with error " << -error_cond << " " << XrdSysE2T(-error_cond)); } } @@ -838,7 +884,7 @@ int File::Read(IO *io, char* iUserBuff, long long iUserOff, int iUserSize) // This can not be skipped as responses write into request memory buffers. if (direct_handler != 0) { - TRACEF(Dump, "File::Read() waiting for direct requests "); + TRACEF(Dump, "Read() waiting for direct requests "); XrdSysCondVarHelper _lck(direct_handler->m_cond); @@ -858,7 +904,7 @@ int File::Read(IO *io, char* iUserBuff, long long iUserOff, int iUserSize) if ( ! error_cond) { error_cond = direct_handler->m_errno; - TRACEF(Error, "File::Read(), direct read finished with error " << -error_cond << " " << XrdSysE2T(-error_cond)); + TRACEF(Error, "Read(), direct read finished with error " << -error_cond << " " << XrdSysE2T(-error_cond)); } } @@ -875,7 +921,7 @@ int File::Read(IO *io, char* iUserBuff, long long iUserOff, int iUserSize) for (BlockList_i bi = blks_processed.begin(); bi != blks_processed.end(); ++bi) { - TRACEF(Dump, "File::Read() dec_ref_count " << (void*)(*bi) << " idx = " << (int)((*bi)->m_offset/BufferSize())); + TRACEF(Dump, "Read() dec_ref_count " << (void*)(*bi) << " idx = " << (int)((*bi)->m_offset/BufferSize())); dec_ref_count(*bi); } @@ -900,20 +946,26 @@ void File::WriteBlockToDisk(Block* b) { // write block buffer into disk file long long offset = b->m_offset - m_offset; - long long size = (offset + m_cfi.GetBufferSize()) > m_file_size ? (m_file_size - offset) : m_cfi.GetBufferSize(); - const char *buff = &b->m_buff[0]; + long long size = b->get_size(); + ssize_t retval; - ssize_t retval = m_data_file->Write(buff, offset, size); + if (m_cfi.IsCkSumCache()) + if (b->has_cksums()) + retval = m_data_file->pgWrite(b->get_buff(), offset, size, b->ref_cksum_vec().data(), b->ref_cksum_vec().size()); + else + retval = m_data_file->pgWrite(b->get_buff(), offset, size, 0, 0); + else + retval = m_data_file->Write(b->get_buff(), offset, size); if (retval < size) { if (retval < 0) { - GetLog()->Emsg("File::WriteToDisk()", -retval, "write block to disk", GetLocalPath().c_str()); + GetLog()->Emsg("WriteToDisk()", -retval, "write block to disk", GetLocalPath().c_str()); } else { - TRACEF(Error, "File::WriteToDisk() incomplete block write ret=" << retval << " (should be " << size << ")"); + TRACEF(Error, "WriteToDisk() incomplete block write ret=" << retval << " (should be " << size << ")"); } XrdSysCondVarHelper _lck(m_state_cond); @@ -926,7 +978,7 @@ void File::WriteBlockToDisk(Block* b) const int blk_idx = (b->m_offset - m_offset) / m_cfi.GetBufferSize(); // Set written bit. - TRACEF(Dump, "File::WriteToDisk() success set bit for block " << b->m_offset << " size=" << size); + TRACEF(Dump, "WriteToDisk() success set bit for block " << b->m_offset << " size=" << size); bool schedule_sync = false; { @@ -935,7 +987,13 @@ void File::WriteBlockToDisk(Block* b) m_cfi.SetBitWritten(blk_idx); if (b->m_prefetch) + { m_cfi.SetBitPrefetch(blk_idx); + } + if (b->req_cksum_net() && ! b->has_cksums() && m_cfi.IsCkSumNet()) + { + m_cfi.ResetCkSumNet(); + } dec_ref_count(b); @@ -969,7 +1027,7 @@ void File::WriteBlockToDisk(Block* b) void File::Sync() { - TRACEF(Dump, "File::Sync()"); + TRACEF(Dump, "Sync()"); int ret = m_data_file->Fsync(); bool errorp = false; @@ -977,23 +1035,23 @@ void File::Sync() { Stats loc_stats = m_stats.Clone(); m_cfi.WriteIOStat(loc_stats); - m_cfi.Write(m_info_file); + m_cfi.Write(m_info_file,m_filename.c_str()); int cret = m_info_file->Fsync(); if (cret != XrdOssOK) { - TRACEF(Error, "File::Sync cinfo file sync error " << cret); + TRACEF(Error, "Sync cinfo file sync error " << cret); errorp = true; } } else { - TRACEF(Error, "File::Sync data file sync error " << ret << ", cinfo file has not been updated"); + TRACEF(Error, "Sync data file sync error " << ret << ", cinfo file has not been updated"); errorp = true; } if (errorp) { - TRACEF(Error, "File::Sync failed, unlinking local files and initiating shutdown of File object"); + TRACEF(Error, "Sync failed, unlinking local files and initiating shutdown of File object"); // Unlink will also call this->initiate_emergency_shutdown() Cache::GetInstance().Unlink(m_filename.c_str()); @@ -1017,7 +1075,7 @@ void File::Sync() m_writes_during_sync.clear(); m_in_sync = false; } - TRACEF(Dump, "File::Sync "<< written_while_in_sync << " blocks written during sync"); + TRACEF(Dump, "Sync "<< written_while_in_sync << " blocks written during sync"); } //------------------------------------------------------------------------------ @@ -1026,7 +1084,7 @@ void File::inc_ref_count(Block* b) { // Method always called under lock. b->m_refcnt++; - TRACEF(Dump, "File::inc_ref_count " << b << " refcnt " << b->m_refcnt); + TRACEF(Dump, "inc_ref_count " << b << " refcnt " << b->m_refcnt); } //------------------------------------------------------------------------------ @@ -1049,12 +1107,12 @@ void File::free_block(Block* b) { // Method always called under lock. int i = b->m_offset / BufferSize(); - TRACEF(Dump, "File::free_block block " << b << " idx = " << i); + TRACEF(Dump, "free_block block " << b << " idx = " << i); size_t ret = m_block_map.erase(i); if (ret != 1) { // assert might be a better option than a warning - TRACEF(Error, "File::free_block did not erase " << i << " from map"); + TRACEF(Error, "free_block did not erase " << i << " from map"); } else { @@ -1119,14 +1177,16 @@ bool File::select_current_io_or_disable_prefetching(bool skip_current) void File::ProcessBlockResponse(BlockResponseHandler* brh, int res) { + static const char* tpfx = "ProcessBlockResponse "; + XrdSysCondVarHelper _lck(m_state_cond); Block *b = brh->m_block; - TRACEF(Dump, "File::ProcessBlockResponse " << (void*)b << " " << b->m_offset/BufferSize()); + TRACEF(Dump, tpfx << "block=" << (void*)b << ", off=" << b->m_offset/BufferSize() << ", res=" << res); // Deregister block from IO's prefetch count, if needed. - if (brh->m_for_prefetch) + if (b->m_prefetch) { IoMap_i mi = m_io_map.find(b->get_io()); if (mi != m_io_map.end()) @@ -1136,7 +1196,7 @@ void File::ProcessBlockResponse(BlockResponseHandler* brh, int res) // If failed and IO is still prefetching -- disable prefetching on this IO. if (res < 0 && mi->second.m_allow_prefetching) { - TRACEF(Debug, "File::ProcessBlockResponse after failed prefetch on io " << b->get_io() << " disabling prefetching on this io."); + TRACEF(Debug, tpfx << "after failed prefetch on io " << b->get_io() << " disabling prefetching on this io."); mi->second.m_allow_prefetching = false; // Check if any IO is still available for prfetching. If not, stop it. @@ -1144,7 +1204,7 @@ void File::ProcessBlockResponse(BlockResponseHandler* brh, int res) { if ( ! select_current_io_or_disable_prefetching(false) ) { - TRACEF(Debug, "ProcessBlockResponse stopping prefetching after io " << b->get_io() << " marked as bad."); + TRACEF(Debug, tpfx << "stopping prefetching after io " << b->get_io() << " marked as bad."); } } } @@ -1157,15 +1217,15 @@ void File::ProcessBlockResponse(BlockResponseHandler* brh, int res) } else { - TRACEF(Error, "File::ProcessBlockResponse io " << b->get_io() << " not found in IoMap."); + TRACEF(Error, tpfx << "io " << b->get_io() << " not found in IoMap."); } } - if (res >= 0) + if (res == b->get_size()) { b->set_downloaded(); // Increase ref-count for the writer. - TRACEF(Dump, "File::ProcessBlockResponse inc_ref_count " << (int)(b->m_offset/BufferSize())); + TRACEF(Dump, tpfx << "inc_ref_count " << (int)(b->m_offset/BufferSize())); if ( ! m_in_shutdown) { inc_ref_count(b); @@ -1175,8 +1235,12 @@ void File::ProcessBlockResponse(BlockResponseHandler* brh, int res) } else { - TRACEF(Error, "File::ProcessBlockResponse block " << b << " " << (int)(b->m_offset/BufferSize()) << " error=" << res); - + if (res < 0) { + TRACEF(Error, tpfx << "block " << b << ", off=" << b->m_offset/BufferSize() << " error=" << res); + } else { + TRACEF(Error, tpfx << "block " << b << ", off=" << b->m_offset/BufferSize() << " incomplete, got " << res << " expected " << b->get_size()); + res = -EREMOTEIO; + } b->set_error(res); } @@ -1213,7 +1277,7 @@ void File::Prefetch() BlockList_t blks; - TRACEF(Dump, "File::Prefetch enter to check download status"); + TRACEF(Dump, "Prefetch enter to check download status"); { XrdSysCondVarHelper _lck(m_state_cond); @@ -1224,12 +1288,12 @@ void File::Prefetch() if ( ! select_current_io_or_disable_prefetching(true) ) { - TRACEF(Error, "File::Prefetch no available IO object found, prefetching stopped. This should not happen, i.e., prefetching should be stopped before."); + TRACEF(Error, "Prefetch no available IO object found, prefetching stopped. This should not happen, i.e., prefetching should be stopped before."); return; } // Select block(s) to fetch. - for (int f = 0; f < m_cfi.GetSizeInBits(); ++f) + for (int f = 0; f < m_cfi.GetNBlocks(); ++f) { if ( ! m_cfi.TestBitWritten(f)) { @@ -1241,7 +1305,7 @@ void File::Prefetch() Block *b = PrepareBlockRequest(f_act, m_current_io->first, true); if (b) { - TRACEF(Dump, "File::Prefetch take block " << f_act); + TRACEF(Dump, "Prefetch take block " << f_act); blks.push_back(b); // Note: block ref_cnt not increased, it will be when placed into write queue. m_prefetch_read_cnt++; @@ -1250,7 +1314,7 @@ void File::Prefetch() else { // This shouldn't happen as prefetching stops when RAM is 70% full. - TRACEF(Warning, "File::Prefetch allocation failed for block " << f_act); + TRACEF(Warning, "Prefetch allocation failed for block " << f_act); } break; } @@ -1259,7 +1323,7 @@ void File::Prefetch() if (blks.empty()) { - TRACEF(Debug, "File::Prefetch file is complete, stopping prefetch."); + TRACEF(Debug, "Prefetch file is complete, stopping prefetch."); m_prefetch_state = kComplete; cache()->DeRegisterPrefetchFile(this); } @@ -1271,7 +1335,7 @@ void File::Prefetch() if ( ! blks.empty()) { - ProcessBlockRequests(blks, true); + ProcessBlockRequests(blks); } } @@ -1293,6 +1357,43 @@ XrdSysTrace* File::GetTrace() return Cache::GetInstance().GetTrace(); } +void File::insert_remote_location(const std::string &loc) +{ + if ( ! loc.empty()) + { + size_t p = loc.find_first_of('@'); + m_remote_locations.insert(&loc[p != string::npos ? p + 1 : 0]); + } +} + +std::string File::GetRemoteLocations() const +{ + std::string s; + if ( ! m_remote_locations.empty()) + { + size_t sl = 0; + int nl = 0; + for (std::set::iterator i = m_remote_locations.begin(); i != m_remote_locations.end(); ++i, ++nl) + { + sl += i->size(); + } + s.reserve(2 + sl + 2*nl + nl - 1 + 1); + s = '['; + int j = 1; + for (std::set::iterator i = m_remote_locations.begin(); i != m_remote_locations.end(); ++i, ++j) + { + s += '"'; s += *i; s += '"'; + if (j < nl) s += ','; + } + s += ']'; + } + else + { + s = "[]"; + } + return s; +} + //============================================================================== //======================= RESPONSE HANDLERS ============================== //============================================================================== diff --git a/src/XrdPfc/XrdPfcFile.hh b/src/XrdPfc/XrdPfcFile.hh index 44db8b5c952..fd803a38f7e 100644 --- a/src/XrdPfc/XrdPfcFile.hh +++ b/src/XrdPfc/XrdPfcFile.hh @@ -70,10 +70,13 @@ public: int m_errno; // stores negative errno bool m_downloaded; bool m_prefetch; + bool m_req_cksum_net; + vCkSum_t m_cksum_vec; - Block(File *f, IO *io, char *buf, long long off, int size, bool m_prefetch) : + Block(File *f, IO *io, char *buf, long long off, int size, bool m_prefetch, bool cks_net) : m_file(f), m_io(io), m_buff(buf), m_offset(off), m_size(size), - m_refcnt(0), m_errno(0), m_downloaded(false), m_prefetch(m_prefetch) + m_refcnt(0), m_errno(0), m_downloaded(false), m_prefetch(m_prefetch), + m_req_cksum_net(cks_net) {} char* get_buff() { return m_buff; } @@ -94,6 +97,10 @@ public: m_errno = 0; m_io = io; } + + bool req_cksum_net() const { return m_req_cksum_net; } + bool has_cksums() const { return ! m_cksum_vec.empty(); } + vCkSum_t& ref_cksum_vec() { return m_cksum_vec; } }; // ================================================================ @@ -102,10 +109,8 @@ class BlockResponseHandler : public XrdOucCacheIOCB { public: Block *m_block; - bool m_for_prefetch; - BlockResponseHandler(Block *b, bool prefetch) : - m_block(b), m_for_prefetch(prefetch) {} + BlockResponseHandler(Block *b) : m_block(b) {} virtual void Done(int result); }; @@ -168,6 +173,11 @@ public: //---------------------------------------------------------------------- bool isOpen() const { return m_is_open; } + //---------------------------------------------------------------------- + //! \brief Notification from IO that it has been updated (remote open). + //---------------------------------------------------------------------- + void ioUpdated(IO *io); + //---------------------------------------------------------------------- //! \brief Initiate close. Return true if still IO active. //! Used in XrdPosixXrootd::Close() @@ -216,10 +226,11 @@ public: Stats DeltaStatsFromLastCall(); + std::string GetRemoteLocations() const; const Info::AStat* GetLastAccessStats() const { return m_cfi.GetLastAccessStats(); } size_t GetAccessCnt() const { return m_cfi.GetAccessCnt(); } int GetBlockSize() const { return m_cfi.GetBufferSize(); } - int GetNBlocks() const { return m_cfi.GetSizeInBits(); } + int GetNBlocks() const { return m_cfi.GetNBlocks(); } int GetNDownloadedBlocks() const { return m_cfi.GetNDownloadedBlocks(); } // These three methods are called under Cache's m_active lock @@ -295,6 +306,9 @@ private: Stats m_stats; //!< cache statistics for this instance Stats m_last_stats; //!< copy of cache stats during last purge cycle, used for per directory stat reporting + std::set m_remote_locations; //!< Gathered in AddIO / ioUpdate / ioActive. + void insert_remote_location(const std::string &loc); + PrefetchState_e m_prefetch_state; int m_prefetch_read_cnt; @@ -317,8 +331,8 @@ private: // Read Block* PrepareBlockRequest(int i, IO *io, bool prefetch); - void ProcessBlockRequest (Block *b, bool prefetch); - void ProcessBlockRequests(BlockList_t& blks, bool prefetch); + void ProcessBlockRequest (Block *b); + void ProcessBlockRequests(BlockList_t& blks); int RequestBlocksDirect(IO *io, DirectResponseHandler *handler, IntList_t& blocks, char* buff, long long req_off, long long req_size); diff --git a/src/XrdPfc/XrdPfcIO.cc b/src/XrdPfc/XrdPfcIO.cc index 9b50c9ead21..59aef49b632 100644 --- a/src/XrdPfc/XrdPfcIO.cc +++ b/src/XrdPfc/XrdPfcIO.cc @@ -3,22 +3,20 @@ using namespace XrdPfc; -IO::IO(XrdOucCacheIO *io, XrdOucCacheStats &stats, Cache &cache) : - m_statsGlobal (stats), +IO::IO(XrdOucCacheIO *io, Cache &cache) : m_cache (cache), m_traceID ("IO"), m_io (io) -{ - m_path = m_io->Path(); -} +{} //============================================================================== void IO::Update(XrdOucCacheIO &iocp) { SetInput(&iocp); - TRACE_PC(Info, const char* loc = m_io->Location(), - "IO::Update() " << Path() << " location: " << + RefreshLocation(); + TRACE_PC(Info, const char* loc = GetLocation(), + "Update() " << Path() << " location: " << ((loc && loc[0] != 0) ? loc : "")); } diff --git a/src/XrdPfc/XrdPfcIO.hh b/src/XrdPfc/XrdPfcIO.hh index 20fcf481c3c..cc54d410b13 100644 --- a/src/XrdPfc/XrdPfcIO.hh +++ b/src/XrdPfc/XrdPfcIO.hh @@ -16,7 +16,7 @@ namespace XrdPfc class IO : public XrdOucCacheIO { public: - IO (XrdOucCacheIO *io, XrdOucCacheStats &stats, Cache &cache); + IO (XrdOucCacheIO *io, Cache &cache); //! Original data source. virtual XrdOucCacheIO *Base() { return m_io; } @@ -45,24 +45,25 @@ public: virtual bool ioActive() = 0; virtual void DetachFinalize() = 0; - XrdSysTrace* GetTrace() { return m_cache.GetTrace(); } + const char* GetLocation() { return m_io->Location(false); } + XrdSysTrace* GetTrace() { return m_cache.GetTrace(); } XrdOucCacheIO* GetInput(); protected: - XrdOucCacheStats &m_statsGlobal; //!< reference to Cache statistics - Cache &m_cache; //!< reference to Cache needed in detach - + Cache &m_cache; //!< reference to Cache needed in detach const char *m_traceID; - std::string m_path; - const char* GetPath() { return m_path.c_str(); } + + const char* GetPath() { return m_io->Path(); } + std::string GetFilename() { return XrdCl::URL(GetPath()).GetPath(); } + const char* RefreshLocation() { return m_io->Location(true); } private: XrdOucCacheIO *m_io; //!< original data source XrdSysMutex updMutex; - void SetInput(XrdOucCacheIO*); + + void SetInput(XrdOucCacheIO*); }; } #endif - diff --git a/src/XrdPfc/XrdPfcIOEntireFile.cc b/src/XrdPfc/XrdPfcIOEntireFile.cc index ed6c56fd05b..74d081edaff 100644 --- a/src/XrdPfc/XrdPfcIOEntireFile.cc +++ b/src/XrdPfc/XrdPfcIOEntireFile.cc @@ -32,14 +32,12 @@ using namespace XrdPfc; //______________________________________________________________________________ -IOEntireFile::IOEntireFile(XrdOucCacheIO *io, XrdOucCacheStats &stats, Cache & cache) : - IO(io, stats, cache), +IOEntireFile::IOEntireFile(XrdOucCacheIO *io, Cache & cache) : + IO(io, cache), m_file(0), m_localStat(0) { - XrdCl::URL url(GetInput()->Path()); - std::string fname = url.GetPath(); - m_file = Cache::GetInstance().GetFile(fname, this); + m_file = Cache::GetInstance().GetFile(GetFilename(), this); } //______________________________________________________________________________ @@ -47,7 +45,7 @@ IOEntireFile::~IOEntireFile() { // called from Detach() if no sync is needed or // from Cache's sync thread - TRACEIO(Debug, "IOEntireFile::~IOEntireFile() " << this); + TRACEIO(Debug, "~IOEntireFile() " << this); delete m_localStat; } @@ -55,9 +53,7 @@ IOEntireFile::~IOEntireFile() //______________________________________________________________________________ int IOEntireFile::Fstat(struct stat &sbuff) { - XrdCl::URL url(GetPath()); - std::string name = url.GetPath(); - name += Info::s_infoExtension; + std::string name = GetFilename() + Info::s_infoExtension; int res = 0; if( ! m_localStat) @@ -81,7 +77,7 @@ int IOEntireFile::initCachedStat(const char* path) { // Called indirectly from the constructor. - static const char* trace_pfx = "IOEntireFile::initCachedStat "; + static const char* trace_pfx = "initCachedStat "; int res = -1; struct stat tmpStat; @@ -128,9 +124,17 @@ int IOEntireFile::initCachedStat(const char* path) return res; } +//______________________________________________________________________________ +void IOEntireFile::Update(XrdOucCacheIO &iocp) +{ + IO::Update(iocp); + m_file->ioUpdated(this); +} + //______________________________________________________________________________ bool IOEntireFile::ioActive() { + RefreshLocation(); return m_file->ioActive(this); } @@ -139,7 +143,7 @@ void IOEntireFile::DetachFinalize() { // Effectively a destructor. - TRACE(Info, "IOEntireFile::DetachFinalize() " << this); + TRACE(Info, "DetachFinalize() " << this); m_file->RequestSyncOfDetachStats(); Cache::GetInstance().ReleaseFile(m_file, this); @@ -150,7 +154,7 @@ void IOEntireFile::DetachFinalize() //______________________________________________________________________________ int IOEntireFile::Read(char *buff, long long off, int size) { - TRACEIO(Dump, "IOEntireFile::Read() "<< this << " off: " << off << " size: " << size); + TRACEIO(Dump, "Read() "<< this << " off: " << off << " size: " << size); // protect from reads over the file size if (off >= FSize()) @@ -175,11 +179,11 @@ int IOEntireFile::Read(char *buff, long long off, int size) // All errors like this should have been already captured by File::Read() // and reflected in its retval. if (size > 0) - TRACEIO(Warning, "IOEntireFile::Read() bytes missed " << size); + TRACEIO(Warning, "Read() bytes missed " << size); } else { - TRACEIO(Warning, "IOEntireFile::Read() error in File::Read(), exit status=" << retval + TRACEIO(Warning, "Read() error in File::Read(), exit status=" << retval << ", error=" << XrdSysE2T(-retval)); } @@ -192,7 +196,6 @@ int IOEntireFile::Read(char *buff, long long off, int size) */ int IOEntireFile::ReadV(const XrdOucIOVec *readV, int n) { - TRACEIO(Dump, "IO::ReadV(), get " << n << " requests" ); + TRACEIO(Dump, "ReadV(), get " << n << " requests" ); return m_file->ReadV(this, readV, n); } - diff --git a/src/XrdPfc/XrdPfcIOEntireFile.hh b/src/XrdPfc/XrdPfcIOEntireFile.hh index 91aae7a3b28..b7779c3fa41 100644 --- a/src/XrdPfc/XrdPfcIOEntireFile.hh +++ b/src/XrdPfc/XrdPfcIOEntireFile.hh @@ -28,7 +28,6 @@ class XrdSysError; class XrdOssDF; -class XfcStats; class XrdOucIOVec; namespace XrdPfc @@ -40,7 +39,7 @@ namespace XrdPfc class IOEntireFile : public IO { public: - IOEntireFile(XrdOucCacheIO *io, XrdOucCacheStats &stats, Cache &cache); + IOEntireFile(XrdOucCacheIO *io, Cache &cache); ~IOEntireFile(); @@ -74,6 +73,8 @@ public: virtual int ReadV(const XrdOucIOVec *readV, int n); + virtual void Update(XrdOucCacheIO &iocp); + //! \brief Abstract virtual method of XrdPfcIO //! Called to check if destruction needs to be done in a separate task. bool ioActive() /* override */; diff --git a/src/XrdPfc/XrdPfcIOFileBlock.cc b/src/XrdPfc/XrdPfcIOFileBlock.cc index c9f3587e13b..413f0ef5219 100644 --- a/src/XrdPfc/XrdPfcIOFileBlock.cc +++ b/src/XrdPfc/XrdPfcIOFileBlock.cc @@ -36,8 +36,8 @@ using namespace XrdPfc; //______________________________________________________________________________ -IOFileBlock::IOFileBlock(XrdOucCacheIO *io, XrdOucCacheStats &statsGlobal, Cache & cache) : - IO(io, statsGlobal, cache), m_localStat(0), m_info(cache.GetTrace(), false), m_info_file(0) +IOFileBlock::IOFileBlock(XrdOucCacheIO *io, Cache & cache) : + IO(io, cache), m_localStat(0), m_info(cache.GetTrace(), false), m_info_file(0) { m_blocksize = Cache::GetInstance().RefConfiguration().m_hdfsbsize; GetBlockSizeFromPath(); @@ -58,11 +58,28 @@ IOFileBlock::~IOFileBlock() // to be held. // I think I need it in ioActive and Read. +//______________________________________________________________________________ +void IOFileBlock::Update(XrdOucCacheIO &iocp) +{ + IO::Update(iocp); + { + XrdSysMutexHelper lock(&m_mutex); + + for (std::map::iterator it = m_blocks.begin(); it != m_blocks.end(); ++it) + { + // Need to update all File / block objects. + if (it->second) it->second->ioUpdated(this); + } + } +} + //______________________________________________________________________________ bool IOFileBlock::ioActive() { // Called from XrdPosixFile when local connection is closed. + RefreshLocation(); + bool active = false; { XrdSysMutexHelper lock(&m_mutex); @@ -85,7 +102,7 @@ void IOFileBlock::DetachFinalize() { // Effectively a destructor. - TRACEIO(Info, "IOFileBlock::DetachFinalize() " << this); + TRACEIO(Info, "DetachFinalize() " << this); CloseInfoFile(); { @@ -116,7 +133,7 @@ void IOFileBlock::CloseInfoFile() Stats as; m_info.WriteIOStatDetach(as); } - m_info.Write(m_info_file); + m_info.Write(m_info_file, GetFilename().c_str()); m_info_file->Fsync(); m_info_file->Close(); @@ -148,7 +165,7 @@ void IOFileBlock::GetBlockSizeFromPath() m_blocksize = atoi(path.substr(pos1).c_str()); } - TRACEIO(Debug, "FileBlock::GetBlockSizeFromPath(), blocksize = " << m_blocksize ); + TRACEIO(Debug, "GetBlockSizeFromPath(), blocksize = " << m_blocksize ); } } @@ -157,8 +174,7 @@ File* IOFileBlock::newBlockFile(long long off, int blocksize) { // NOTE: Can return 0 if opening of a local file fails! - XrdCl::URL url(GetInput()->Path()); - std::string fname = url.GetPath(); + std::string fname = GetFilename(); std::stringstream ss; ss << fname; @@ -168,9 +184,9 @@ File* IOFileBlock::newBlockFile(long long off, int blocksize) ss << &offExt[0]; fname = ss.str(); - TRACEIO(Debug, "FileBlock::FileBlock(), create XrdPfcFile "); + TRACEIO(Debug, "FileBlock(), create XrdPfcFile "); - File* file = Cache::GetInstance().GetFile(fname, this, off, blocksize); + File *file = Cache::GetInstance().GetFile(fname, this, off, blocksize); return file; } @@ -197,9 +213,7 @@ long long IOFileBlock::FSize() //______________________________________________________________________________ int IOFileBlock::initLocalStat() { - XrdCl::URL url(GetPath()); - std::string path = url.GetPath(); - path += ".cinfo"; + std::string path = GetFilename() + Info::s_infoExtension; int res = -1; struct stat tmpStat; @@ -211,16 +225,16 @@ int IOFileBlock::initLocalStat() m_info_file = m_cache.GetOss()->newFile(m_cache.RefConfiguration().m_username.c_str()); if (m_info_file->Open(path.c_str(), O_RDWR, 0600, myEnv) == XrdOssOK) { - if (m_info.Read(m_info_file, path)) + if (m_info.Read(m_info_file, path.c_str())) { tmpStat.st_size = m_info.GetFileSize(); - TRACEIO(Info, "IOFileBlock::initCachedStat successfuly read size from existing info file = " << tmpStat.st_size); + TRACEIO(Info, "initCachedStat successfuly read size from existing info file = " << tmpStat.st_size); res = 0; } else { // file exist but can't read it - TRACEIO(Debug, "IOFileBlock::initCachedStat info file is not complete"); + TRACEIO(Debug, "initCachedStat info file is not complete"); } } } @@ -231,7 +245,7 @@ int IOFileBlock::initLocalStat() if (m_info_file) { delete m_info_file; m_info_file = 0; } res = GetInput()->Fstat(tmpStat); - TRACEIO(Debug, "IOFileBlock::initCachedStat get stat from client res= " << res << "size = " << tmpStat.st_size); + TRACEIO(Debug, "initCachedStat get stat from client res= " << res << "size = " << tmpStat.st_size); if (res == 0) { if (m_cache.GetOss()->Create(m_cache.RefConfiguration().m_username.c_str(), path.c_str(), 0600, myEnv, XRDOSS_mkpath) == XrdOssOK) @@ -245,17 +259,17 @@ int IOFileBlock::initLocalStat() m_info.SetBufferSize(m_cache.RefConfiguration().m_bufferSize); m_info.DisableDownloadStatus(); m_info.SetFileSize(tmpStat.st_size); - m_info.Write(m_info_file, path); + m_info.Write(m_info_file, path.c_str()); m_info_file->Fsync(); } else { - TRACEIO(Error, "IOFileBlock::initCachedStat can't open info file path"); + TRACEIO(Error, "initCachedStat can't open info file path"); } } else { - TRACEIO(Error, "IOFileBlock::initCachedStat can't create info file path"); + TRACEIO(Error, "initCachedStat can't create info file path"); } } } @@ -289,7 +303,7 @@ int IOFileBlock::Read(char *buff, long long off, int size) int idx_first = off0 / m_blocksize; int idx_last = (off0 + size - 1) / m_blocksize; int bytes_read = 0; - TRACEIO(Dump, "IOFileBlock::Read() "<< off << "@" << size << " block range ["<< idx_first << ", " << idx_last << "]"); + TRACEIO(Dump, "Read() "<< off << "@" << size << " block range ["<< idx_first << ", " << idx_last << "]"); for (int blockIdx = idx_first; blockIdx <= idx_last; ++blockIdx) { @@ -309,7 +323,7 @@ int IOFileBlock::Read(char *buff, long long off, int size) if (blockIdx == lastIOFileBlock ) { pbs = fileSize - blockIdx*m_blocksize; - // TRACEIO(Dump, "IOFileBlock::Read() last block, change output file size to " << pbs); + // TRACEIO(Dump, "Read() last block, change output file size to " << pbs); } // Note: File* can be 0 and stored as 0 if local open fails! @@ -338,13 +352,13 @@ int IOFileBlock::Read(char *buff, long long off, int size) } } - TRACEIO(Dump, "IOFileBlock::Read() block[ " << blockIdx << "] read-block-size[" << readBlockSize << "], offset[" << readBlockSize << "] off = " << off ); + TRACEIO(Dump, "Read() block[ " << blockIdx << "] read-block-size[" << readBlockSize << "], offset[" << readBlockSize << "] off = " << off ); int retvalBlock = (fb != 0) ? fb->Read(this, buff, off, readBlockSize) : GetInput()->Read(buff, off, readBlockSize); - TRACEIO(Dump, "IOFileBlock::Read() Block read returned " << retvalBlock); + TRACEIO(Dump, "Read() Block read returned " << retvalBlock); if (retvalBlock == readBlockSize) { bytes_read += retvalBlock; @@ -353,12 +367,12 @@ int IOFileBlock::Read(char *buff, long long off, int size) } else if (retvalBlock >= 0) { - TRACEIO(Warning, "IOFileBlock::Read() incomplete read, missing bytes " << readBlockSize-retvalBlock); + TRACEIO(Warning, "Read() incomplete read, missing bytes " << readBlockSize-retvalBlock); return -EIO; } else { - TRACEIO(Error, "IOFileBlock::Read() read error, retval" << retvalBlock); + TRACEIO(Error, "Read() read error, retval" << retvalBlock); return retvalBlock; } } diff --git a/src/XrdPfc/XrdPfcIOFileBlock.hh b/src/XrdPfc/XrdPfcIOFileBlock.hh index 1a4a795f0fa..b56afac26c4 100644 --- a/src/XrdPfc/XrdPfcIOFileBlock.hh +++ b/src/XrdPfc/XrdPfcIOFileBlock.hh @@ -38,7 +38,7 @@ namespace XrdPfc class IOFileBlock : public IO { public: - IOFileBlock(XrdOucCacheIO *io, XrdOucCacheStats &stats, Cache &cache); + IOFileBlock(XrdOucCacheIO *io, Cache &cache); ~IOFileBlock(); @@ -61,6 +61,8 @@ public: virtual long long FSize(); + virtual void Update(XrdOucCacheIO &iocp); + private: long long m_blocksize; //!< size of file-block std::map m_blocks; //!< map of created blocks diff --git a/src/XrdPfc/XrdPfcInfo.cc b/src/XrdPfc/XrdPfcInfo.cc index f4529126863..b8d4e14178b 100644 --- a/src/XrdPfc/XrdPfcInfo.cc +++ b/src/XrdPfc/XrdPfcInfo.cc @@ -24,6 +24,7 @@ #include #include "XrdOss/XrdOss.hh" +#include "XrdOuc/XrdOucCRC32C.hh" #include "XrdCks/XrdCksCalcmd5.hh" #include "XrdSys/XrdSysTrace.hh" #include "XrdCl/XrdClLog.hh" @@ -35,20 +36,37 @@ namespace { + +struct TraceHeader +{ + const char *f_tpf, *f_tdir, *f_tfile , *f_tmsg; + + // tdir is supposed to be '/' terminated. Check can be added in ctor and a bool flag to mark if it is needed. + TraceHeader(const char *tpf, const char *tdir, const char *tfile = 0, const char *tmsg = 0) : + f_tpf(tpf), f_tdir(tdir), f_tfile(tfile), f_tmsg(tmsg) {} +}; + +XrdSysTrace& operator<<(XrdSysTrace& s, const TraceHeader& th) +{ + s << th.f_tpf << " " << th.f_tdir; + if (th.f_tfile) s << th.f_tfile; + if (th.f_tmsg) s << " " << th.f_tmsg; + s << " "; + return s; +} + struct FpHelper { XrdOssDF *f_fp; off_t f_off; XrdSysTrace *f_trace; const char *m_traceID; - std::string f_ttext; + const TraceHeader &f_trace_hdr; XrdSysTrace* GetTrace() const { return f_trace; } - FpHelper(XrdOssDF* fp, off_t off, - XrdSysTrace *trace, const char *tid, const std::string &ttext) : - f_fp(fp), f_off(off), - f_trace(trace), m_traceID(tid), f_ttext(ttext) + FpHelper(XrdOssDF* fp, off_t off, XrdSysTrace *trace, const char *tid, const TraceHeader &thdr) : + f_fp(fp), f_off(off), f_trace(trace), m_traceID(tid), f_trace_hdr(thdr) {} // Returns true on error @@ -59,8 +77,8 @@ struct FpHelper { if (warnp) { - TRACE(Warning, f_ttext << " off=" << f_off << " size=" << size - << " ret=" << ret << " error=" << ((ret < 0) ? XrdSysE2T(-ret) : "")); + TRACE(Warning, f_trace_hdr << "Oss Read failed at off=" << f_off << " size=" << size + << " ret=" << ret << " error=" << ((ret < 0) ? XrdSysE2T(-ret) : "")); } return true; } @@ -79,8 +97,8 @@ struct FpHelper ssize_t ret = f_fp->Write(buf, f_off, size); if (ret != size) { - TRACE(Warning, f_ttext << " off=" << f_off << " size=" << size - << " ret=" << ret << " error=" << ((ret < 0) ? XrdSysE2T(ret) : "")); + TRACE(Warning, f_trace_hdr << "Oss Write failed at off=" << f_off << " size=" << size + << " ret=" << ret << " error=" << ((ret < 0) ? XrdSysE2T(ret) : "")); return true; } f_off += ret; @@ -96,10 +114,11 @@ struct FpHelper using namespace XrdPfc; -const char* Info::m_traceID = "CInfo"; -const char* Info::s_infoExtension = ".cinfo"; -const int Info::s_defaultVersion = 3; - size_t Info::s_maxNumAccess = 20; // default, can be changed through configuration +const char* Info::m_traceID = "CInfo"; +const char* Info::s_infoExtension = ".cinfo"; +const size_t Info::s_infoExtensionLen = strlen(Info::s_infoExtension); + size_t Info::s_maxNumAccess = 20; // default, can be changed through configuration +const int Info::s_defaultVersion = 4; //------------------------------------------------------------------------------ @@ -109,7 +128,7 @@ Info::Info(XrdSysTrace* trace, bool prefetchBuffer) : m_buff_written(0), m_buff_prefetch(0), m_sizeInBits(0), m_complete(false), - m_cksCalc(0) + m_cksCalcMd5(0) {} Info::~Info() @@ -117,7 +136,7 @@ Info::~Info() if (m_store.m_buff_synced) free(m_store.m_buff_synced); if (m_buff_written) free(m_buff_written); if (m_buff_prefetch) free(m_buff_prefetch); - delete m_cksCalc; + delete m_cksCalcMd5; } //------------------------------------------------------------------------------ @@ -183,15 +202,117 @@ void Info::ResizeBits(int s) //------------------------------------------------------------------------------ -bool Info::Read(XrdOssDF* fp, const std::string &fname) +void Info::DisableDownloadStatus() +{ + // use version sign to skip download status + m_store.m_version = -m_store.m_version; +} + +//------------------------------------------------------------------------------ + +void Info::ResetCkSumCache() +{ + if (IsCkSumCache()) + { + m_store.m_status.f_cksum_check &= ~CSChk_Cache; + if ( ! HasNoCkSumTime()) + m_store.m_noCkSumTime = time(0); + } +} + +void Info::ResetCkSumNet() +{ + if (IsCkSumNet()) + { + m_store.m_status.f_cksum_check &= ~CSChk_Net; + if ( ! HasNoCkSumTime()) + m_store.m_noCkSumTime = time(0); + } +} + +//------------------------------------------------------------------------------ +// Write / Read cinfo file +//------------------------------------------------------------------------------ + +uint32_t Info::GetCksum() +{ + uint32_t cks = crc32c(0, &m_store, 24); + return crc32c(cks, m_store.m_buff_synced, GetSizeInBytes()); +} + +void Info::GetCksumMd5(unsigned char* buff, char* digest) +{ + if (m_cksCalcMd5) + m_cksCalcMd5->Init(); + else + m_cksCalcMd5 = new XrdCksCalcmd5(); + + m_cksCalcMd5->Update((const char*)buff, GetSizeInBytes()); + memcpy(digest, m_cksCalcMd5->Final(), 16); +} + +const char* Info::GetCkSumStateAsText() const { - // does not need lock, called only in File::Open - // before File::Run() starts + switch (m_store.m_status.f_cksum_check) { + case CSChk_None : return "none"; + case CSChk_Cache : return "cache"; + case CSChk_Net : return "net"; + case CSChk_Both : return "both"; + default : return "unknown"; + } +} - std::string trace_pfx("Info:::Read() "); - trace_pfx += fname + " "; +//------------------------------------------------------------------------------ + +// std::string wrapper ? +// bool Info::Write(XrdOssDF* fp, const std::string &fname) +// {} - FpHelper r(fp, 0, m_trace, m_traceID, trace_pfx + "oss read failed"); +bool Info::Write(XrdOssDF* fp, const char *dname, const char *fname) +{ + TraceHeader trace_pfx(":Write()", dname, fname); + + if (m_store.m_astats.size() > s_maxNumAccess) CompactifyAccessRecords(); + + FpHelper w(fp, 0, m_trace, m_traceID, trace_pfx); + + m_store.m_version = s_defaultVersion; + if (w.Write(m_store.m_version)) return false; + if (w.Write(m_store.m_status._raw_)) return false; + if (w.Write(m_store.m_buffer_size)) return false; + if (w.Write(m_store.m_file_size)) return false; + + if (w.WriteRaw(m_store.m_buff_synced, GetSizeInBytes())) return false; + + m_store.m_cksum = GetCksum(); + if (w.Write(m_store.m_cksum)) return false; + + if (w.Write(m_store.m_creationTime)) return false; + if (w.Write(m_store.m_noCkSumTime)) return false; + + if (w.Write(m_store.m_accessCnt)) return false; + for (std::vector::iterator it = m_store.m_astats.begin(); it != m_store.m_astats.end(); ++it) + { + if (w.Write(*it)) return false; + } + + return true; +} + +//------------------------------------------------------------------------------ + +// Potentially provide std::string wrapper. +// bool Info::Read(XrdOssDF* fp, const std::string &fname) +// {} + +bool Info::Read(XrdOssDF *fp, const char *dname, const char *fname) +{ + // Does not need lock, called only in File::Open before File::Run() starts. + // XXXX Wait, how about Purge, and LocalFilePath, Stat? + + TraceHeader trace_pfx(":Read()", dname, fname); + + FpHelper r(fp, 0, m_trace, m_traceID, trace_pfx); if (r.Read(m_store.m_version)) return false; @@ -199,21 +320,25 @@ bool Info::Read(XrdOssDF* fp, const std::string &fname) { if (abs(m_store.m_version) == 1) { - return ReadV1(fp, fname); + return ReadV1(fp, dname, fname); } else if (m_store.m_version == 2) { - return ReadV2(fp, fname); + return ReadV2(fp, dname, fname); + } + else if (m_store.m_version == 3) + { + return ReadV3(fp, dname, fname); } else { - TRACE(Warning, trace_pfx << " File version " << m_store.m_version << " not supported."); + TRACE(Warning, trace_pfx << "File version " << m_store.m_version << " not supported."); return false; } } - if (r.Read(m_store.m_buffer_size)) return false; - + if (r.Read(m_store.m_status._raw_)) return false; + if (r.Read(m_store.m_buffer_size)) return false; long long fs; if (r.Read(fs)) return false; SetFileSize(fs); @@ -221,31 +346,22 @@ bool Info::Read(XrdOssDF* fp, const std::string &fname) if (r.ReadRaw(m_store.m_buff_synced, GetSizeInBytes())) return false; memcpy(m_buff_written, m_store.m_buff_synced, GetSizeInBytes()); - if (r.ReadRaw(m_store.m_cksum, 16)) return false; - char tmpCksum[16]; - GetCksum(&m_store.m_buff_synced[0], &tmpCksum[0]); - - // Debug print cksum: - // for (int i =0; i < 16; ++i) - // printf("%x", tmpCksum[i] & 0xff); - // for (int i =0; i < 16; ++i) - // printf("%x", m_store.m_cksum[i] & 0xff); - - if (memcmp(m_store.m_cksum, &tmpCksum[0], 16)) + if (r.Read(m_store.m_cksum)) return false; + if (GetCksum() != m_store.m_cksum) { - TRACE(Error, trace_pfx << " buffer cksum and saved cksum don't match \n"); + TRACE(Error, trace_pfx << "Buffer cksum and saved cksum don't match \n"); return false; } // cache complete status m_complete = ! IsAnythingEmptyInRng(0, m_sizeInBits); - // read creation time + // read creation and no-cksum times if (r.Read(m_store.m_creationTime)) return false; + if (r.Read(m_store.m_noCkSumTime)) return false; // get number of accessess if (r.Read(m_store.m_accessCnt, false)) m_store.m_accessCnt = 0; // was: return false; - TRACE(Dump, trace_pfx << " complete "<< m_complete << " access_cnt " << m_store.m_accessCnt); // read access statistics m_store.m_astats.reserve(std::min(m_store.m_accessCnt, s_maxNumAccess)); @@ -259,58 +375,7 @@ bool Info::Read(XrdOssDF* fp, const std::string &fname) } //------------------------------------------------------------------------------ - -void Info::GetCksum( unsigned char* buff, char* digest) -{ - if (m_cksCalc) - m_cksCalc->Init(); - else - m_cksCalc = new XrdCksCalcmd5(); - - m_cksCalc->Update((const char*)buff, GetSizeInBytes()); - memcpy(digest, m_cksCalc->Final(), 16); -} - -//------------------------------------------------------------------------------ - -void Info::DisableDownloadStatus() -{ - // use version sign to skip download status - m_store.m_version = -m_store.m_version; -} - -//------------------------------------------------------------------------------ - -bool Info::Write(XrdOssDF* fp, const std::string &fname) -{ - std::string trace_pfx("Info:::Write() "); - trace_pfx += fname + " "; - - if (m_store.m_astats.size() > s_maxNumAccess) CompactifyAccessRecords(); - - FpHelper w(fp, 0, m_trace, m_traceID, trace_pfx + "oss write failed"); - - m_store.m_version = s_defaultVersion; - if (w.Write(m_store.m_version)) return false; - if (w.Write(m_store.m_buffer_size)) return false; - if (w.Write(m_store.m_file_size)) return false; - - if (w.WriteRaw(m_store.m_buff_synced, GetSizeInBytes())) return false; - - GetCksum(&m_store.m_buff_synced[0], &m_store.m_cksum[0]); - if (w.Write(m_store.m_cksum)) return false; - - if (w.Write(m_store.m_creationTime)) return false; - - if (w.Write(m_store.m_accessCnt)) return false; - for (std::vector::iterator it = m_store.m_astats.begin(); it != m_store.m_astats.end(); ++it) - { - if (w.Write(*it)) return false; - } - - return true; -} - +// Access stats / records //------------------------------------------------------------------------------ void Info::ResetAllAccessStats() @@ -451,7 +516,57 @@ const Info::AStat* Info::GetLastAccessStats() const // Support for reading of previous cinfo versions //============================================================================== -bool Info::ReadV2(XrdOssDF* fp, const std::string &fname) +bool Info::ReadV3(XrdOssDF* fp, const char *dname, const char *fname) +{ + TraceHeader trace_pfx(":ReadV3()", dname, fname); + + FpHelper r(fp, 0, m_trace, m_traceID, trace_pfx); + + if (r.Read(m_store.m_version)) return false; + if (r.Read(m_store.m_buffer_size)) return false; + + long long fs; + if (r.Read(fs)) return false; + SetFileSize(fs); + + if (r.ReadRaw(m_store.m_buff_synced, GetSizeInBytes())) return false; + memcpy(m_buff_written, m_store.m_buff_synced, GetSizeInBytes()); + + char fileCksum[16], tmpCksum[16]; + if (r.ReadRaw(&fileCksum[0], 16)) return false; + GetCksumMd5(&m_store.m_buff_synced[0], &tmpCksum[0]); + + if (memcmp(&fileCksum[0], &tmpCksum[0], 16)) + { + TRACE(Error, trace_pfx << "buffer cksum and saved cksum don't match \n"); + return false; + } + + // cache complete status + m_complete = ! IsAnythingEmptyInRng(0, m_sizeInBits); + + // read creation time + if (r.Read(m_store.m_creationTime)) return false; + + // get number of accessess + if (r.Read(m_store.m_accessCnt, false)) m_store.m_accessCnt = 0; // was: return false; + + // read access statistics + m_store.m_astats.reserve(std::min(m_store.m_accessCnt, s_maxNumAccess)); + AStat as; + while ( ! r.Read(as, false)) + { + as.Reserved = 0; + m_store.m_astats.emplace_back(as); + } + + // Comment for V4: m_store.m_noCkSumTime and m_store_mstatus.f_cksum_check + // are left as 0 (default values in Info ctor). + + return true; +} + +bool Info::ReadV2(XrdOssDF* fp, const char *dname, const char *fname) { struct AStatV2 { @@ -462,12 +577,11 @@ bool Info::ReadV2(XrdOssDF* fp, const std::string &fname) long long BytesBypassed; //! read remote client }; - std::string trace_pfx("Info:::ReadV2() "); - trace_pfx += fname + " "; + TraceHeader trace_pfx(":ReadV2()", dname, fname); - FpHelper r(fp, 0, m_trace, m_traceID, trace_pfx + "oss read failed"); + FpHelper r(fp, 0, m_trace, m_traceID, trace_pfx); - if (r.Read(m_store.m_version)) return false; + if (r.Read(m_store.m_version)) return false; if (r.Read(m_store.m_buffer_size)) return false; long long fs; @@ -477,19 +591,13 @@ bool Info::ReadV2(XrdOssDF* fp, const std::string &fname) if (r.ReadRaw(m_store.m_buff_synced, GetSizeInBytes())) return false; memcpy(m_buff_written, m_store.m_buff_synced, GetSizeInBytes()); - if (r.ReadRaw(m_store.m_cksum, 16)) return false; - char tmpCksum[16]; - GetCksum(&m_store.m_buff_synced[0], &tmpCksum[0]); + char fileCksum[16], tmpCksum[16]; + if (r.ReadRaw(&fileCksum[0], 16)) return false; + GetCksumMd5(&m_store.m_buff_synced[0], &tmpCksum[0]); - // Debug print cksum: - // for (int i =0; i < 16; ++i) - // printf("%x", tmpCksum[i] & 0xff); - // for (int i =0; i < 16; ++i) - // printf("%x", m_store.m_cksum[i] & 0xff); - - if (strncmp(m_store.m_cksum, &tmpCksum[0], 16)) + if (memcmp(&fileCksum[0], &tmpCksum[0], 16)) { - TRACE(Error, trace_pfx << " buffer cksum and saved cksum don't match \n"); + TRACE(Error, trace_pfx << "buffer cksum and saved cksum don't match \n"); return false; } @@ -501,20 +609,19 @@ bool Info::ReadV2(XrdOssDF* fp, const std::string &fname) // get number of accessess if (r.Read(m_store.m_accessCnt, false)) m_store.m_accessCnt = 0; // was: return false; - TRACE(Dump, trace_pfx << " complete "<< m_complete << " access_cnt " << m_store.m_accessCnt); // read access statistics m_store.m_astats.reserve(std::min(m_store.m_accessCnt, s_maxNumAccess)); AStatV2 av2; while ( ! r.ReadRaw(&av2, sizeof(AStatV2), false)) { - AStat as; as.AttachTime = av2.AttachTime; as.DetachTime = av2.DetachTime; as.NumIos = 1; as.Duration = av2.DetachTime - av2.AttachTime; as.NumMerged = 0; + as.Reserved = 0; as.BytesHit = av2.BytesHit; as.BytesMissed = av2.BytesMissed; as.BytesBypassed = av2.BytesBypassed; @@ -527,7 +634,7 @@ bool Info::ReadV2(XrdOssDF* fp, const std::string &fname) //------------------------------------------------------------------------------ -bool Info::ReadV1(XrdOssDF* fp, const std::string &fname) +bool Info::ReadV1(XrdOssDF* fp, const char *dname, const char *fname) { struct AStatV1 { @@ -537,12 +644,11 @@ bool Info::ReadV1(XrdOssDF* fp, const std::string &fname) long long BytesBypassed; //! read remote client }; - std::string trace_pfx("Info:::ReadV1() "); - trace_pfx += fname + " "; + TraceHeader trace_pfx(":ReadV1()", dname, fname); - FpHelper r(fp, 0, m_trace, m_traceID, trace_pfx + "oss read failed"); + FpHelper r(fp, 0, m_trace, m_traceID, trace_pfx); - if (r.Read(m_store.m_version)) return false; + if (r.Read(m_store.m_version)) return false; if (r.Read(m_store.m_buffer_size)) return false; long long fs; @@ -554,7 +660,6 @@ bool Info::ReadV1(XrdOssDF* fp, const std::string &fname) m_complete = ! IsAnythingEmptyInRng(0, m_sizeInBits); if (r.ReadRaw(&m_store.m_accessCnt, sizeof(int), false)) m_store.m_accessCnt = 0; // was: return false; - TRACE(Dump, trace_pfx << " complete "<< m_complete << " access_cnt " << m_store.m_accessCnt); m_store.m_astats.reserve(std::min(m_store.m_accessCnt, s_maxNumAccess)); AStatV1 av1; @@ -566,6 +671,7 @@ bool Info::ReadV1(XrdOssDF* fp, const std::string &fname) as.NumIos = 1; as.Duration = 0; as.NumMerged = 0; + as.Reserved = 0; as.BytesHit = av1.BytesHit; as.BytesMissed = av1.BytesMissed; as.BytesBypassed = av1.BytesBypassed; @@ -580,3 +686,44 @@ bool Info::ReadV1(XrdOssDF* fp, const std::string &fname) return true; } + + +//============================================================================== +// Test bitfield ops and masking of non-cksum fields +//============================================================================== +#ifdef XRDPFC_CKSUM_TEST + +void Info::TestCksumStuff() +{ + static const char* names[] = { "--", "-C", "N-", "NC" }; + + const Configuration &conf = Cache::GetInstance().RefConfiguration(); + + printf("Doing cksum tests for config %s\n", names[conf.m_cs_Chk]); + + Info cfi(0); + printf("cksum %d, raw %x\n", cfi.m_store.m_status.f_cksum_check, cfi.m_store.m_status._raw_); + + cfi.SetCkSumState(CSChk_Both); + printf("cksum %d, raw %x\n", cfi.m_store.m_status.f_cksum_check, cfi.m_store.m_status._raw_); + + cfi.m_store.m_status._raw_ |= 0xff0000; + printf("cksum %d, raw %x\n", cfi.m_store.m_status.f_cksum_check, cfi.m_store.m_status._raw_); + + cfi.ResetCkSumCache(); + printf("cksum %d, raw %x\n", cfi.m_store.m_status.f_cksum_check, cfi.m_store.m_status._raw_); + + cfi.ResetCkSumNet(); + printf("cksum %d, raw %x\n", cfi.m_store.m_status.f_cksum_check, cfi.m_store.m_status._raw_); + + for (int cs = CSChk_None; cs <= CSChk_Both; ++cs) + { + cfi.SetCkSumState((CkSumCheck_e) cs); + bool hasmb = conf.does_cschk_have_missing_bits(cfi.GetCkSumState()); + cfi.DowngradeCkSumState(conf.get_cs_Chk()); + printf("-- File conf %s -- does_cschk_have_missing_bits:%d, downgraded_state:%s\n", + names[cs], hasmb, names[cfi.GetCkSumState()]); + } +} + +#endif \ No newline at end of file diff --git a/src/XrdPfc/XrdPfcInfo.hh b/src/XrdPfc/XrdPfcInfo.hh index 695a6f2b510..99bd7d0e7d1 100644 --- a/src/XrdPfc/XrdPfcInfo.hh +++ b/src/XrdPfc/XrdPfcInfo.hh @@ -27,16 +27,12 @@ #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClDefaultEnv.hh" +#include "XrdPfcTypes.hh" + class XrdOssDF; class XrdCksCalc; class XrdSysTrace; - -namespace XrdCl -{ -class Log; -} - namespace XrdPfc { class Stats; @@ -48,6 +44,18 @@ class Stats; class Info { public: + struct Status { + union { + struct { + int f_cksum_check : 3; //!< as in enum CkSumCheck_e + + int _free_bits_ : 29; + }; + unsigned int _raw_; + }; + Status() : _raw_(0) {} + }; + //! Access statistics struct AStat { @@ -56,12 +64,13 @@ public: int NumIos; //!< number of IO objects attached during this access int Duration; //!< total duration of all IOs attached int NumMerged; //!< number of times the record has been merged + int Reserved; //!< reserved / alignment long long BytesHit; //!< read from cache long long BytesMissed; //!< read from remote and cached long long BytesBypassed; //!< read from remote and dropped AStat() : - AttachTime(0), DetachTime(0), NumIos(0), Duration(0), NumMerged(0), + AttachTime(0), DetachTime(0), NumIos(0), Duration(0), NumMerged(0), Reserved(0), BytesHit(0), BytesMissed(0), BytesBypassed(0) {} @@ -70,16 +79,21 @@ public: struct Store { - int m_version; //!< info version + int m_version; //!< info file version + Status m_status; //!< status information long long m_buffer_size; //!< buffer / block size long long m_file_size; //!< size of file in bytes unsigned char *m_buff_synced; //!< disk written state vector - char m_cksum[16]; //!< cksum of downloaded information + uint32_t m_cksum; //!< cksum of downloaded information time_t m_creationTime; //!< time the info file was created + time_t m_noCkSumTime; //!< time when first non-cksummed block was detected size_t m_accessCnt; //!< total access count for the file std::vector m_astats; //!< access records - Store () : m_version(1), m_buffer_size(-1), m_file_size(0), m_buff_synced(0), m_creationTime(0), m_accessCnt(0) {} + Store () : + m_version(1), m_buffer_size(-1), m_file_size(0), m_buff_synced(0), + m_creationTime(0), m_noCkSumTime(0), m_accessCnt(0) + {} }; @@ -129,26 +143,27 @@ public: //--------------------------------------------------------------------- //! \brief Reserve buffer for file_size / buffer_size bytes - //! //! @param n number of file blocks //--------------------------------------------------------------------- void ResizeBits(int n); //--------------------------------------------------------------------- - //! \brief Rea load content from cinfo file into this object - //! + //! \brief Read content of cinfo file into this object //! @param fp file handle - //! @param fname optional file name for trace output - //! + //! @param dname directory name for trace output + //! @param fname optional file name for trace output (can be included in dname) //! @return true on success //--------------------------------------------------------------------- - bool Read(XrdOssDF* fp, const std::string &fname = ""); + bool Read(XrdOssDF* fp, const char *dname, const char *fname = 0); //--------------------------------------------------------------------- //! Write number of blocks and read buffer size + //! @param fp file handle + //! @param dname directory name for trace output + //! @param fname optional file name for trace output (can be included in dname) //! @return true on success //--------------------------------------------------------------------- - bool Write(XrdOssDF* fp, const std::string &fname = ""); + bool Write(XrdOssDF* fp, const char *dname, const char *fname = 0); //--------------------------------------------------------------------- //! Compactify access records to the configured maximum. @@ -203,7 +218,7 @@ public: //--------------------------------------------------------------------- //! Get number of blocks represented in download-state bit-vector. //--------------------------------------------------------------------- - int GetSizeInBits() const; + int GetNBlocks() const; //--------------------------------------------------------------------- //! Get file size @@ -271,14 +286,42 @@ public: const Store& RefStoredData() const { return m_store; } //--------------------------------------------------------------------- - //! Get md5 cksum + //! Get file size //--------------------------------------------------------------------- - void GetCksum( unsigned char* buff, char* digest); + time_t GetCreationTime() const { return m_store.m_creationTime; } + + //--------------------------------------------------------------------- + //! Get cksum, MD5 is for backward compatibility with V2 and V3. + //--------------------------------------------------------------------- + uint32_t GetCksum(); + void GetCksumMd5(unsigned char* buff, char* digest); + + CkSumCheck_e GetCkSumState() const { return (CkSumCheck_e) m_store.m_status.f_cksum_check; } + const char* GetCkSumStateAsText() const; + + bool IsCkSumCache() const { return m_store.m_status.f_cksum_check & CSChk_Cache; } + bool IsCkSumNet() const { return m_store.m_status.f_cksum_check & CSChk_Net; } + bool IsCkSumAny() const { return m_store.m_status.f_cksum_check & CSChk_Both; } + bool IsCkSumBoth() const { return (m_store.m_status.f_cksum_check & CSChk_Both) == CSChk_Both; } + + void SetCkSumState(CkSumCheck_e css) { m_store.m_status.f_cksum_check = css; } + void DowngradeCkSumState(CkSumCheck_e css_ref) { m_store.m_status.f_cksum_check &= css_ref; } + void ResetCkSumCache(); + void ResetCkSumNet(); + + bool HasNoCkSumTime() const { return m_store.m_noCkSumTime != 0; } + time_t GetNoCkSumTime() const { return m_store.m_noCkSumTime; } + time_t GetNoCkSumTimeForUVKeep() const { return m_store.m_noCkSumTime ? m_store.m_noCkSumTime : m_store.m_creationTime; } + +#ifdef XRDPFC_CKSUM_TEST + static void TestCksumStuff(); +#endif static const char* m_traceID; // has to be m_ (convention in TRACE macros) static const char* s_infoExtension; - static const int s_defaultVersion; + static const size_t s_infoExtensionLen; static size_t s_maxNumAccess; // can be set from configuration + static const int s_defaultVersion; XrdSysTrace* GetTrace() const {return m_trace; } @@ -297,10 +340,11 @@ private: inline unsigned char cfiBIT(int n) const { return 1 << n; } // Reading functions for older cinfo file formats - bool ReadV1(XrdOssDF* fp, const std::string &fname); - bool ReadV2(XrdOssDF* fp, const std::string &fname); + bool ReadV1(XrdOssDF* fp, const char *dname, const char *fname); + bool ReadV2(XrdOssDF* fp, const char *dname, const char *fname); + bool ReadV3(XrdOssDF* fp, const char *dname, const char *fname); - XrdCksCalc* m_cksCalc; + XrdCksCalc* m_cksCalcMd5; }; //------------------------------------------------------------------------------ @@ -395,7 +439,7 @@ inline int Info::GetSizeInBytes() const return 0; } -inline int Info::GetSizeInBits() const +inline int Info::GetNBlocks() const { return m_sizeInBits; } diff --git a/src/XrdPfc/XrdPfcPrint.cc b/src/XrdPfc/XrdPfcPrint.cc index 63e772afd6f..4f35fc075a7 100644 --- a/src/XrdPfc/XrdPfcPrint.cc +++ b/src/XrdPfc/XrdPfcPrint.cc @@ -66,40 +66,43 @@ void Print::printFile(const std::string& path) XrdSysTrace tr(""); tr.What = 2; Info cfi(&tr); - if ( ! cfi.Read(fh, path)) + if ( ! cfi.Read(fh, path.c_str())) { return; } int cntd = 0; - for (int i = 0; i < cfi.GetSizeInBits(); ++i) + for (int i = 0; i < cfi.GetNBlocks(); ++i) { if (cfi.TestBitWritten(i)) cntd++; } const Info::Store& store = cfi.RefStoredData(); - char creationBuff[1000]; - time_t creationTime = store.m_creationTime; - strftime(creationBuff, 1000, "%c", localtime(&creationTime)); + char timeBuff[128]; + strftime(timeBuff, 128, "%c", localtime(&store.m_creationTime)); - printf("version %d, created %s\n", cfi.GetVersion(), creationBuff); + printf("version %d, created %s; cksum %s", cfi.GetVersion(), timeBuff, cfi.GetCkSumStateAsText()); + if (cfi.HasNoCkSumTime()) { + strftime(timeBuff, 128, "%c", localtime(&store.m_noCkSumTime)); + printf(", no-cksum-time %s\n", timeBuff); + } else printf("\n"); printf("file_size %lld kB, buffer_size %lld kB, n_blocks %d, n_downloaded %d, state %scomplete [%.3f%%]\n", - cfi.GetFileSize() >> 10, cfi.GetBufferSize() >> 10, cfi.GetSizeInBits(), cntd, - (cntd < cfi.GetSizeInBits()) ? "in" : "", 100.0 * cntd / cfi.GetSizeInBits()); + cfi.GetFileSize() >> 10, cfi.GetBufferSize() >> 10, cfi.GetNBlocks(), cntd, + (cntd < cfi.GetNBlocks()) ? "in" : "", 100.0 * cntd / cfi.GetNBlocks()); if (m_verbose) { int8_t n_db = 0; - { int x = cfi.GetSizeInBits(); while (x) + { int x = cfi.GetNBlocks(); while (x) { x /= 10; ++n_db; } } static const char *nums = "0123456789"; - printf("printing %d blocks:\n", cfi.GetSizeInBits()); + printf("printing %d blocks:\n", cfi.GetNBlocks()); printf("%*s %10d%10d%10d%10d%10d%10d\n", n_db, "", 1, 2, 3, 4, 5, 6); printf("%*s %s%s%s%s%s%s0123", n_db, "", nums, nums, nums, nums, nums, nums); - for (int i = 0; i < cfi.GetSizeInBits(); ++i) + for (int i = 0; i < cfi.GetNBlocks(); ++i) { if (i % 64 == 0) printf("\n%*d ", n_db, i); diff --git a/src/XrdPfc/XrdPfcPurge.cc b/src/XrdPfc/XrdPfcPurge.cc index a64d0bf844c..6e513164143 100644 --- a/src/XrdPfc/XrdPfcPurge.cc +++ b/src/XrdPfc/XrdPfcPurge.cc @@ -5,6 +5,7 @@ #include #include "XrdOuc/XrdOucEnv.hh" +#include "XrdOss/XrdOssAt.hh" #include "XrdSys/XrdSysTrace.hh" using namespace XrdPfc; @@ -18,6 +19,10 @@ XrdSysTrace* GetTrace() return Cache::GetInstance().GetTrace(); } +// Temporary, extensive purge tracing +// #define TRACE_PURGE(x) TRACE(Debug, x) +// #define TRACE_PURGE(x) std::cout << "PURGE " << x << "\n" +#define TRACE_PURGE(x) //============================================================================== // DirState @@ -39,8 +44,8 @@ class DirState // quota info, enabled? int m_depth; - int m_max_depth; - bool m_stat_report; // not used - storing of stats required + int m_max_depth; // XXXX Do we need this? Should it be passed in to find functions? + bool m_stat_report; // not used yet - storing of stats requested typedef std::map DsMap_t; typedef DsMap_t::iterator DsMap_i; @@ -178,7 +183,7 @@ class DataFsState public: DataFsState() : - m_max_depth ( Cache::GetInstance().RefConfiguration().m_dirStatsStoreDepth ), + m_max_depth ( Cache::Conf().m_dirStatsStoreDepth ), m_root ( m_max_depth ), m_prev_time ( time(0) ) {} @@ -222,10 +227,10 @@ class FPurgeState std::string path; long long nBytes; time_t time; - DirState *dirState; // XXXX if this is stored, why is it not used later in purge? + DirState *dirState; - FS(const std::string& p, long long n, time_t t, DirState *ds) : - path(p), nBytes(n), time(t), dirState(ds) + FS(const std::string &dname, const char *fname, long long n, time_t t, DirState *ds) : + path(dname + fname), nBytes(n), time(t), dirState(ds) {} }; @@ -243,67 +248,72 @@ class FPurgeState long long nBytesAccum; long long nBytesTotal; time_t tMinTimeStamp; + time_t tMinUVKeepTimeStamp; + + // XrdOss *m_oss; + XrdOssAt m_oss_at; // ------------------------------------ // Directory handling & stat collection // ------------------------------------ DirState *m_dir_state; + std::string m_current_path; // Includes trailing '/' int m_dir_level; - int m_max_dir_level_for_stat_collection; // until I honor globs from pfc.dirstats - std::string m_current_dir; - std::string m_current_path; // Note: without leading '/'! + const int m_max_dir_level_for_stat_collection; // until we honor globs from pfc.dirstats std::vector m_dir_names_stack; std::vector m_dir_usage_stack; - void begin_traversal(DirState *root) + const char *m_info_ext; + const size_t m_info_ext_len; + XrdSysTrace *m_trace; + + static const char *m_traceID; + + + void begin_traversal(DirState *root, const char *root_path = "/") { m_dir_state = root; m_dir_level = 0; - m_max_dir_level_for_stat_collection = Cache::GetInstance().RefConfiguration().m_dirStatsStoreDepth; - m_current_dir = ""; - m_current_path = ""; - m_dir_names_stack.reserve(32); - m_dir_usage_stack.reserve(m_max_dir_level_for_stat_collection + 1); + m_current_path = root_path; m_dir_usage_stack.push_back(0); - printf("FPurgeState::begin_traversal cur_path '%s', usage=%lld, level=%d\n", m_current_path.c_str(), - m_dir_usage_stack.back(), m_dir_level); + TRACE_PURGE("FPurgeState::begin_traversal cur_path '" << m_current_path << "', usage=" << m_dir_usage_stack.back() << ", level=" << m_dir_level); } void end_traversal() { - printf("FPurgeState::end_traversal reporting for '%s', usage=%lld, nBytesTotal=%lld, level=%d\n", m_current_path.c_str(), - m_dir_usage_stack.back(), nBytesTotal, m_dir_level); + TRACE_PURGE("FPurgeState::end_traversal reporting for '" << m_current_path << "', usage=" << m_dir_usage_stack.back() << ", nBytesTotal=" << nBytesTotal << ", level=" << m_dir_level); m_dir_state->set_usage(m_dir_usage_stack.back()); m_dir_state = 0; } - void cd_down(const std::string& dir_name, const std::string& full_path) + void cd_down(const std::string& dir_name) { ++m_dir_level; + if (m_dir_level <= m_max_dir_level_for_stat_collection) { m_dir_usage_stack.push_back(0); m_dir_state = m_dir_state->find_dir(dir_name, true); } + m_dir_names_stack.push_back(dir_name); - m_current_dir = dir_name; - m_current_path = full_path; + m_current_path.append(dir_name); + m_current_path.append("/"); } - void cd_up(const std::string& full_path) + void cd_up() { if (m_dir_level <= m_max_dir_level_for_stat_collection) { long long tail = m_dir_usage_stack.back(); m_dir_usage_stack.pop_back(); - printf("FPurgeState::cd_up reporting for '%s', usage=%lld, level=%d\n", m_current_path.c_str(), - tail, m_dir_level); + TRACE_PURGE("FPurgeState::cd_up reporting for '" << m_current_path << "', usage=" << tail << ", level=" << m_dir_level); m_dir_state->set_usage(tail); m_dir_state = m_dir_state->get_parent(); @@ -311,8 +321,8 @@ class FPurgeState m_dir_usage_stack.back() += tail; } - m_current_path = full_path; - m_current_dir = m_dir_names_stack.back(); + // remove trailing / and last dir but keep the new trailing / in place. + m_current_path.erase(m_current_path.find_last_of('/', m_current_path.size() - 2) + 1); m_dir_names_stack.pop_back(); --m_dir_level; @@ -321,16 +331,26 @@ class FPurgeState // ------------------------------------------------------------------------ // ------------------------------------------------------------------------ - FPurgeState(long long iNBytesReq) : - nBytesReq(iNBytesReq), nBytesAccum(0), nBytesTotal(0), tMinTimeStamp(0), - m_dir_state(0) - {} + FPurgeState(long long iNBytesReq, XrdOss &oss) : + nBytesReq(iNBytesReq), nBytesAccum(0), nBytesTotal(0), tMinTimeStamp(0), tMinUVKeepTimeStamp(0), + // m_oss(oss), + m_oss_at(oss), + m_dir_state(0), m_dir_level(0), + m_max_dir_level_for_stat_collection(Cache::Conf().m_dirStatsStoreDepth), + m_info_ext(XrdPfc::Info::s_infoExtension), + m_info_ext_len(strlen(XrdPfc::Info::s_infoExtension)), + m_trace(Cache::GetInstance().GetTrace()) + { + m_current_path.reserve(256); + m_dir_names_stack.reserve(32); + m_dir_usage_stack.reserve(m_max_dir_level_for_stat_collection + 1); + } // ------------------------------------------------------------------------ void setMinTime(time_t min_time) { tMinTimeStamp = min_time; } time_t getMinTime() const { return tMinTimeStamp; } - + void setUVKeepMinTime(time_t min_time) { tMinUVKeepTimeStamp = min_time; } long long getNBytesTotal() const { return nBytesTotal; } void MoveListEntriesToMap() @@ -342,21 +362,78 @@ class FPurgeState m_flist.clear(); } - void checkFile(const std::string& iPath, long long iNBytes, time_t iTime) + /* + void UnlinkInfoAndData(const char *fname, long long nbytes, XrdOssDF *iOssDF) + { + fname[fname_len - m_info_ext_len] = 0; + if (nbytes > 0) + { + if ( ! Cache.GetInstance().IsFileActiveOrPurgeProtected(dataPath)) + { + m_n_purged++; + m_bytes_purged += nbytes; + } else + { + m_n_purge_protected++; + m_bytes_purge_protected += nbytes; + m_dir_state->add_usage_purged(nbytes); + // XXXX should also tweak other stuff? + fname[fname_len - m_info_ext_len] = '.'; + return; + } + } + m_oss_at.Unlink(*iOssDF, fname); + fname[fname_len - m_info_ext_len] = '.'; + m_oss_at.Unlink(*iOssDF, fname); + } + */ + + void CheckFile(const char *fname, Info &info, struct stat &fstat /*, XrdOssDF *iOssDF*/) { - nBytesTotal += iNBytes; + static const char *trc_pfx = "FPurgeState::CheckFile "; + + long long nbytes = info.GetNDownloadedBytes(); + time_t atime; + if ( ! info.GetLatestDetachTime(atime)) + { + // cinfo file does not contain any known accesses, use fstat.mtime instead. + TRACE(Debug, trc_pfx << "could not get access time for " << m_current_path << fname << ", using mtime from stat instead."); + atime = fstat.st_mtime; + } + // TRACE(Dump, trc_pfx << "checking " << fname << " accessTime " << atime); + + nBytesTotal += nbytes; + + m_dir_usage_stack.back() += nbytes; + + // XXXX Should remove aged-out files here ... but I have trouble getting + // the DirState and purge report set up consistently. + // Need some serious code reorganization here. + // Biggest problem is maintaining overall state a traversal state consistently. + // Sigh. - if (m_dir_state) m_dir_usage_stack.back() += iNBytes; + // In first two cases we lie about FS time (set to 0) to get them all removed early. + // The age-based purge atime would also be good as there should be nothing + // before that time in the map anyway. + // But we use 0 as a test in purge loop to make sure we continue even if enough + // disk-space has been freed. - if (tMinTimeStamp > 0 && iTime < tMinTimeStamp) + if (tMinTimeStamp > 0 && atime < tMinTimeStamp) { - m_flist.push_back(FS(iPath, iNBytes, iTime, m_dir_state)); - nBytesAccum += iNBytes; + m_flist.push_back(FS(m_current_path, fname, nbytes, 0, m_dir_state)); + nBytesAccum += nbytes; } - else if (nBytesAccum < nBytesReq || ( ! m_fmap.empty() && iTime < m_fmap.rbegin()->first)) + else if (tMinUVKeepTimeStamp > 0 && + Cache::Conf().does_cschk_have_missing_bits(info.GetCkSumState()) && + info.GetNoCkSumTimeForUVKeep() < tMinUVKeepTimeStamp) { - m_fmap.insert(std::make_pair(iTime, FS(iPath, iNBytes, iTime, m_dir_state))); - nBytesAccum += iNBytes; + m_flist.push_back(FS(m_current_path, fname, nbytes, 0, m_dir_state)); + nBytesAccum += nbytes; + } + else if (nBytesAccum < nBytesReq || ( ! m_fmap.empty() && atime < m_fmap.rbegin()->first)) + { + m_fmap.insert(std::make_pair(atime, FS(m_current_path, fname, nbytes, atime, m_dir_state))); + nBytesAccum += nbytes; // remove newest files from map if necessary while ( ! m_fmap.empty() && nBytesAccum - m_fmap.rbegin()->second.nBytes >= nBytesReq) @@ -367,102 +444,75 @@ class FPurgeState } } - void FillFileMapRecurse(XrdOssDF* iOssDF, const std::string& path) + void TraverseNamespace(XrdOssDF *iOssDF) { - static const char* m_traceID = "Purge"; - - const char *InfoExt = XrdPfc::Info::s_infoExtension; - const size_t InfoExtLen = strlen(InfoExt); - - Cache &cache = Cache::GetInstance(); - XrdOss *oss = cache.GetOss(); - const char *uname = cache.RefConfiguration().m_username.c_str(); + static const char *trc_pfx = "FPurgeState::TraverseNamespace "; char fname[256]; + struct stat fstat; XrdOucEnv env; - while (iOssDF->Readdir(&fname[0], 256) >= 0) - { - // printf("readdir [%s]\n", fname); + TRACE_PURGE("Starting to read dir [" << m_current_path << "], iOssDF->getFD()=" << iOssDF->getFD() << "."); - std::string new_path = path + "/"; new_path += fname; - size_t fname_len = strlen(&fname[0]); + iOssDF->StatRet(&fstat); - if (fname_len == 0) - { - // std::cout << "Finish read dir.[" << new_path << "] Break loop.\n"; + while (iOssDF->Readdir(fname, 256) >= 0) + { + TRACE_PURGE(" Readdir [" << fname << "]"); + + if (fname[0] == 0) { + TRACE_PURGE(" Finished reading dir [" << m_current_path << "]. Break loop."); break; } + if (fname[0] == '.' && (fname[1] == 0 || (fname[1] == '.' && fname[2] == 0))) { + TRACE_PURGE(" Skipping here or parent dir [" << fname << "]. Continue loop."); + continue; + } - if (strncmp("..", &fname[0], 2) && strncmp(".", &fname[0], 1)) - { - XrdOssDF* dh = oss->newDir (uname); - XrdOssDF* fh = oss->newFile(uname); + size_t fname_len = strlen(fname); + XrdOssDF *dfh = 0; - if (fname_len > InfoExtLen && strncmp(&fname[fname_len - InfoExtLen], InfoExt, InfoExtLen) == 0) + if (S_ISDIR(fstat.st_mode)) + { + if (m_oss_at.Opendir(*iOssDF, fname, env, dfh) == XrdOssOK) { - // Check if the file is currently opened / purge-protected is done before unlinking of the file. - - Info cinfo(cache.GetTrace()); - - if (fh->Open(new_path.c_str(), O_RDONLY, 0600, env) == XrdOssOK && cinfo.Read(fh, new_path)) - { - bool all_gauda = true; - time_t accessTime; - if ( ! cinfo.GetLatestDetachTime(accessTime)) - { - // cinfo file does not contain any known accesses, use stat.mtime instead. - TRACE(Debug, "FillFileMapRecurse() could not get access time for " << new_path << ", trying stat"); - - struct stat fstat; - if (oss->Stat(new_path.c_str(), &fstat) == XrdOssOK) - { - accessTime = fstat.st_mtime; - TRACE(Dump, "FillFileMapRecurse() have access time for " << new_path << " via stat: " << accessTime); - } - else - { - // This really shouldn't happen ... but if it does remove cinfo and the data file right away. - TRACE(Warning, "FillFileMapRecurse() could not get access time for " << new_path << "; purging."); - oss->Unlink(new_path.c_str()); - new_path = new_path.substr(0, new_path.size() - strlen(InfoExt)); - oss->Unlink(new_path.c_str()); - all_gauda = false; - } - } - - if (all_gauda) - { - // TRACE(Dump, "FillFileMapRecurse() checking " << fname << " accessTime " << accessTime); - checkFile(new_path, cinfo.GetNDownloadedBytes(), accessTime); - } - } - else - { - TRACE(Warning, "FillFileMapRecurse() can't open or read " << new_path << ", err " << XrdSysE2T(errno) - << "; purging."); - oss->Unlink(new_path.c_str()); - new_path = new_path.substr(0, new_path.size() - InfoExtLen); - oss->Unlink(new_path.c_str()); - } + cd_down(fname); + TraverseNamespace(dfh); + cd_up(); } - else if (dh->Opendir(new_path.c_str(), env) == XrdOssOK) - { - if (m_dir_state) cd_down(fname, new_path); + else + TRACE(Warning, trc_pfx << "could not opendir [" << m_current_path << fname << "], " << XrdSysE2T(errno)); + } + else if (fname_len > m_info_ext_len && strncmp(&fname[fname_len - m_info_ext_len], m_info_ext, m_info_ext_len) == 0) + { + // Check if the file is currently opened / purge-protected is done before unlinking of the file. - FillFileMapRecurse(dh, new_path); + Info cinfo(m_trace); - if (m_dir_state) cd_up(path); + if (m_oss_at.OpenRO(*iOssDF, fname, env, dfh) == XrdOssOK && cinfo.Read(dfh, m_current_path.c_str(), fname)) + { + CheckFile(fname, cinfo, fstat); } - - delete dh; dh = 0; - delete fh; fh = 0; + else + { + TRACE(Warning, trc_pfx << "can't open or read " << m_current_path << fname << ", err " << XrdSysE2T(errno) << "; purging."); + m_oss_at.Unlink(*iOssDF, fname); + fname[fname_len - m_info_ext_len] = 0; + m_oss_at.Unlink(*iOssDF, fname); + } + } + else // XXXX devel debug only, to be removed + { + TRACE_PURGE(" Ignoring [" << fname << "], not a dir or cinfo."); } + + delete dfh; } } - }; +const char *FPurgeState::m_traceID = "Purge"; + //============================================================================== // ResourceMonitor @@ -503,7 +553,7 @@ class ScanAndPurgeJob : public XrdJob void Cache::copy_out_active_stats_and_update_data_fs_state() { - static const char *trc_pfx = "Cache::copy_out_active_stats_and_update_data_fs_state() "; + static const char *trc_pfx = "copy_out_active_stats_and_update_data_fs_state() "; StatsMMap_t updates; { @@ -518,7 +568,7 @@ void Cache::copy_out_active_stats_and_update_data_fs_state() } } - m_fs_state->reset_stats(); + m_fs_state->reset_stats(); // XXXX-CKSUM rethink how to do this if we keep some purge entries for next time for (StatsMMap_i i = updates.begin(); i != updates.end(); ++i) { @@ -541,7 +591,7 @@ void Cache::copy_out_active_stats_and_update_data_fs_state() void Cache::ResourceMonitorHeartBeat() { - // static const char *trc_pfx = "Cache::ResourceMonitorHeartBeat() "; + // static const char *trc_pfx = "ResourceMonitorHeartBeat() "; // Pause before initial run sleep(1); @@ -633,7 +683,7 @@ void Cache::ResourceMonitorHeartBeat() void Cache::Purge() { - static const char *trc_pfx = "Cache::Purge() "; + static const char *trc_pfx = "Purge() "; XrdOucEnv env; long long disk_usage; @@ -642,7 +692,7 @@ void Cache::Purge() // Pause before initial run sleep(1); - if (m_configuration.are_dirstats_enabled()) m_fs_state = new DataFsState; + m_fs_state = new DataFsState; // { PathTokenizer p("/a/b/c/f.root", 2, true); p.deboog(); } // { PathTokenizer p("/a/b/f.root", 2, true); p.deboog(); } @@ -714,25 +764,21 @@ void Cache::Purge() long long bytesToRemove = std::max(bytesToRemove_d, bytesToRemove_f); bool enforce_age_based_purge = false; - if (m_configuration.is_age_based_purge_in_effect()) + if (m_configuration.is_age_based_purge_in_effect() || m_configuration.is_uvkeep_purge_in_effect()) { + // XXXX ... I could collect those guys in larger vectors (maps?) and do traversal when + // they are empty. if (--age_based_purge_countdown <= 0) { enforce_age_based_purge = true; - age_based_purge_countdown = m_configuration.m_purgeColdFilesPeriod; + age_based_purge_countdown = m_configuration.m_purgeAgeBasedPeriod; } } - bool enforce_traversal_for_usage_collection = false; - - if (m_fs_state) - { - copy_out_active_stats_and_update_data_fs_state(); - - enforce_traversal_for_usage_collection = is_first; + bool enforce_traversal_for_usage_collection = is_first; + // XXX Other conditions? Periodic checks? - // XXX Other conditions? Periodic checks? - } + copy_out_active_stats_and_update_data_fs_state(); TRACE(Debug, trc_pfx << "Precheck:"); TRACE(Debug, "\tbytes_to_remove_disk = " << bytesToRemove_d << " B"); @@ -746,7 +792,9 @@ void Cache::Purge() bool purge_required = (bytesToRemove > 0 || enforce_age_based_purge); - FPurgeState purgeState(2 * bytesToRemove); // prepare twice more volume than required + // XXXX-PurgeOpt Need to retain this state between purges so I can avoid doing + // the traversal more often than really needed. + FPurgeState purgeState(2 * bytesToRemove, *m_oss); // prepare twice more volume than required if (purge_required || enforce_traversal_for_usage_collection) { @@ -756,21 +804,19 @@ void Cache::Purge() { purgeState.setMinTime(time(0) - m_configuration.m_purgeColdFilesAge); } + if (m_configuration.is_uvkeep_purge_in_effect()) + { + purgeState.setUVKeepMinTime(time(0) - m_configuration.m_cs_UVKeep); + } XrdOssDF* dh = m_oss->newDir(m_configuration.m_username.c_str()); - if (dh->Opendir("", env) == XrdOssOK) + if (dh->Opendir("/", env) == XrdOssOK) { - if (m_fs_state) - { - purgeState.begin_traversal(m_fs_state->get_root()); - } + purgeState.begin_traversal(m_fs_state->get_root()); - purgeState.FillFileMapRecurse(dh, ""); + purgeState.TraverseNamespace(dh); - if (m_fs_state) - { - purgeState.end_traversal(); - } + purgeState.end_traversal(); dh->Close(); } @@ -819,7 +865,8 @@ void Cache::Purge() } // Dump statistcs before actual purging so maximum usage values get recorded. - if (m_fs_state) + // Should really go to gstream --- and should really go from Heartbeat. + if (m_configuration.is_dir_stat_reporting_on()) { m_fs_state->dump_recursively(); } @@ -828,18 +875,20 @@ void Cache::Purge() { // Loop over map and remove files with oldest values of access time. struct stat fstat; + size_t info_ext_len = strlen(Info::s_infoExtension); int protected_cnt = 0; long long protected_sum = 0; for (FPurgeState::map_i it = purgeState.m_fmap.begin(); it != purgeState.m_fmap.end(); ++it) { - // Finish when enough space has been freed but not while purging of cold files is in progress. - if (bytesToRemove <= 0 && ! (m_configuration.is_age_based_purge_in_effect() && it->first < purgeState.getMinTime())) + // Finish when enough space has been freed but not while age-based purging is in progress. + // Those files are marked with time-stamp = 0. + if (bytesToRemove <= 0 && ! (enforce_age_based_purge && it->first == 0)) { break; } - std::string infoPath = it->second.path; - std::string dataPath = infoPath.substr(0, infoPath.size() - strlen(XrdPfc::Info::s_infoExtension)); + std::string &infoPath = it->second.path; + std::string dataPath = infoPath.substr(0, infoPath.size() - info_ext_len); if (IsFileActiveOrPurgeProtected(dataPath)) { @@ -872,24 +921,18 @@ void Cache::Purge() m_oss->Unlink(dataPath.c_str()); TRACE(Dump, trc_pfx << "Removed file: '" << dataPath << "' size: " << it->second.nBytes << ", time: " << it->first); - if (m_fs_state) - { - DirState *ds = m_fs_state->find_dirstate_for_lfn(dataPath); - if (ds != 0) - ds->add_usage_purged(it->second.nBytes); - else - TRACE(Error, trc_pfx << "Failed finding DirState for file '" << dataPath << "'."); - } + if (it->second.dirState != 0) // XXXX This should now always be true. + it->second.dirState->add_usage_purged(it->second.nBytes); + else + TRACE(Error, trc_pfx << "DirState not set for file '" << dataPath << "'."); } } if (protected_cnt > 0) { TRACE(Info, trc_pfx << "Encountered " << protected_cnt << " protected files, sum of their size: " << protected_sum); } - if (m_fs_state) - { - m_fs_state->upward_propagate_usage_purged(); - } + + m_fs_state->upward_propagate_usage_purged(); } { @@ -912,39 +955,4 @@ void Cache::Purge() } } - -//============================================================================== -// DirStats specific stuff -//============================================================================== - -/* - - RRDtool DB sketch - - # --start is not needed, default is "now - 10s" - - rrdtool create .crrd --step \ - DS:open_events:ABSOLUTE:<2*purge_interval>:0:1000 \ - DS:access_duration:ABSOLUTE:<2*purge_interval>:0:1000000 \ - DS:bytes_disk:ABSOLUTE:<2*purge_interval>:0:100000000000 \ - DS:bytes_fetch:ABSOLUTE:<2*purge_interval>:0:100000000000 \ - DS:bytes_bypass:ABSOLUTE:<2*purge_interval>:0:100000000000 \ - DS:bytes_served:COMPUTE:bytes_disk,bytes_fetch,bytes_bypass,+,+ \ - RRA:AVERAGE:0.5:s:1w \ - RRA:AVERAGE:0.5:1h:1M \ - RRA:AVERAGE:0.5:1d:1y \ - - Questions / Issues: - 1. DS min / max -- are they for values after division or before? - I'm assuming after in the above. - 2. What is xxf (argument to RRA)? The thing that is usually 0.5. - Ah, xfiles factor, how much can be unknown for consolidated value to eb known. - 3. Use COMPUTE for bytes_served (sum of the others). - ARGH, can not change heartbeat for COMPUTE, will pass it in manually. - 4. Use rddtool tune -h to change heartbeat on start (if different, maybe). - - 5. ADD disk_usage !!!! - DS:disk_usage:GAUGE:... - */ - } // end XrdPfc namespace diff --git a/src/XrdPfc/XrdPfcTrace.hh b/src/XrdPfc/XrdPfcTrace.hh index 8faaf8e9dc8..c891c44805a 100644 --- a/src/XrdPfc/XrdPfcTrace.hh +++ b/src/XrdPfc/XrdPfcTrace.hh @@ -38,27 +38,27 @@ #define TRACE(act, x) \ if (XRD_TRACE What >= TRACE_ ## act) \ - SYSTRACE(XRD_TRACE, 0, m_traceID, 0, TRACE_STR_ ## act << x) + SYSTRACE(XRD_TRACE, 0, m_traceID, 0, TRACE_STR_ ## act << x) #define TRACE_INT(act, x) \ if (XRD_TRACE What >= act) \ {static const char* t_what[]={"","error ","warning ","info ","debug ","dump "};\ - SYSTRACE(XRD_TRACE, 0, m_traceID, 0, t_what[act] <= TRACE_ ## act) \ - {pre_code; SYSTRACE(XRD_TRACE, 0, m_traceID, 0, TRACE_STR_ ## act <= TRACE_ ## act) SYSTRACE(XRD_TRACE, 0, m_traceID, 0, \ - TRACE_STR_ ## act <= TRACE_ ## act) SYSTRACE(XRD_TRACE, 0, m_traceID, 0, \ - TRACE_STR_ ## act <. +//---------------------------------------------------------------------------------- + +namespace XrdPfc +{ +enum CkSumCheck_e { CSChk_Unknown = -1, CSChk_None = 0, CSChk_Cache = 1, CSChk_Net = 2, CSChk_Both = 3, + CSChk_TLS = 4 // Only used during configuration parsing. +}; + +typedef std::vector vCkSum_t; +} + +// #define XRDPFC_CKSUM_TEST + +#endif diff --git a/src/XrdPfc/XrdPfcVRead.cc b/src/XrdPfc/XrdPfcVRead.cc index 5c4a9b9b96a..2ffddbe58ab 100644 --- a/src/XrdPfc/XrdPfcVRead.cc +++ b/src/XrdPfc/XrdPfcVRead.cc @@ -80,7 +80,7 @@ using namespace XrdPfc; int File::ReadV(IO *io, const XrdOucIOVec *readV, int n) { - TRACEF(Dump, "File::ReadV for " << n << " chunks."); + TRACEF(Dump, "ReadV for " << n << " chunks."); if ( ! VReadValidate(readV, n)) { @@ -103,7 +103,7 @@ int File::ReadV(IO *io, const XrdOucIOVec *readV, int n) if ( ! m_is_open) { m_state_cond.UnLock(); - TRACEF(Error, "File::ReadV file is not open"); + TRACEF(Error, "ReadV file is not open"); return io->GetInput()->ReadV(readV, n); } @@ -122,7 +122,7 @@ int File::ReadV(IO *io, const XrdOucIOVec *readV, int n) // request blocks that need to be fetched if (bytesRead >= 0) { - ProcessBlockRequests(blks_to_request, false); + ProcessBlockRequests(blks_to_request); } // issue a client read @@ -361,7 +361,7 @@ int File::VReadProcessBlocks(IO *io, const XrdOucIOVec *readV, int n, { if (bi->block->is_failed() && bi->block->get_io() != io) { - TRACEF(Info, "File::VReadProcessBlocks() requested block " << bi->block << " failed with another io " << + TRACEF(Info, "VReadProcessBlocks() requested block " << bi->block << " failed with another io " << bi->block->get_io() << " - reissuing request with my io " << io); bi->block->reset_error_and_set_io(io); @@ -387,7 +387,7 @@ int File::VReadProcessBlocks(IO *io, const XrdOucIOVec *readV, int n, } } - ProcessBlockRequests(to_reissue, false); + ProcessBlockRequests(to_reissue); to_reissue.clear(); std::vector::iterator bi = finished.begin(); @@ -419,7 +419,7 @@ int File::VReadProcessBlocks(IO *io, const XrdOucIOVec *readV, int n, else { bytes_read = bi->block->m_errno; - TRACEF(Error, "File::VReadProcessBlocks() io " << io << ", block "<< bi->block << + TRACEF(Error, "VReadProcessBlocks() io " << io << ", block "<< bi->block << " finished with error " << -bytes_read << " " << XrdSysE2T(-bytes_read)); break; }