From 30500b0ca67dee6956f6d4a4d8a2f6384bb6513c Mon Sep 17 00:00:00 2001 From: Matevz Tadel Date: Tue, 8 May 2018 15:13:14 -0700 Subject: [PATCH] Make sure per file access stats are synchronized properly. --- src/XrdFileCache/XrdFileCacheFile.cc | 13 +++++++---- src/XrdFileCache/XrdFileCacheIOFileBlock.cc | 1 + src/XrdFileCache/XrdFileCacheStats.hh | 20 +++++++++++----- src/XrdFileCache/XrdFileCacheVRead.cc | 26 +++++++++++++++------ 4 files changed, 43 insertions(+), 17 deletions(-) diff --git a/src/XrdFileCache/XrdFileCacheFile.cc b/src/XrdFileCache/XrdFileCacheFile.cc index 98f69833a61..2d8298b2fca 100644 --- a/src/XrdFileCache/XrdFileCacheFile.cc +++ b/src/XrdFileCache/XrdFileCacheFile.cc @@ -171,7 +171,8 @@ bool File::FinalizeSyncBeforeExit() { if ( ! m_writes_during_sync.empty() || m_non_flushed_cnt > 0 || ! m_detachTimeIsLogged) { - m_cfi.WriteIOStatDetach(m_stats); + Stats loc_stats = m_stats.Clone(); + m_cfi.WriteIOStatDetach(loc_stats); m_detachTimeIsLogged = true; TRACEF(Debug, "File::FinalizeSyncBeforeExit scheduling sync to write detach stats"); return true; @@ -443,7 +444,6 @@ int File::ReadBlocksFromDisk(std::list& blocks, total += rs; } - m_stats.m_BytesDisk += total; return total; } @@ -458,6 +458,8 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) const long long BS = m_cfi.GetBufferSize(); + Stats loc_stats; + // lock // loop over reqired blocks: // - if on disk, ok; @@ -564,6 +566,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) if (rc >= 0) { bytes_read += rc; + loc_stats.m_BytesDisk += rc; } else { @@ -620,7 +623,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) TRACEF(Dump, "File::Read() ub=" << (void*)iUserBuff << " from finished block " << (*bi)->m_offset/BS << " size " << size_to_copy); memcpy(&iUserBuff[user_off], &((*bi)->m_buff[off_in_block]), size_to_copy); bytes_read += size_to_copy; - m_stats.m_BytesRam += size_to_copy; + loc_stats.m_BytesRam += size_to_copy; if ((*bi)->m_prefetch) prefetchHitsRam++; } @@ -653,7 +656,7 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) if (direct_handler->m_errno == 0) { bytes_read += direct_size; - m_stats.m_BytesMissed += direct_size; + loc_stats.m_BytesMissed += direct_size; } else { @@ -688,6 +691,8 @@ int File::Read(char* iUserBuff, long long iUserOff, int iUserSize) m_prefetchScore = float(m_prefetchHitCnt)/m_prefetchReadCnt; } + m_stats.AddStats(loc_stats); + return bytes_read; } diff --git a/src/XrdFileCache/XrdFileCacheIOFileBlock.cc b/src/XrdFileCache/XrdFileCacheIOFileBlock.cc index 93d5e45bb67..5f24726e3ab 100644 --- a/src/XrdFileCache/XrdFileCacheIOFileBlock.cc +++ b/src/XrdFileCache/XrdFileCacheIOFileBlock.cc @@ -88,6 +88,7 @@ void IOFileBlock::CloseInfoFile() { if (m_info.GetFileSize() > 0) { + // We do not maintain access statistics for individual blocks. Stats as; m_info.WriteIOStatDetach(as); } diff --git a/src/XrdFileCache/XrdFileCacheStats.hh b/src/XrdFileCache/XrdFileCacheStats.hh index 312344e4aed..a09b24f86a0 100644 --- a/src/XrdFileCache/XrdFileCacheStats.hh +++ b/src/XrdFileCache/XrdFileCacheStats.hh @@ -27,7 +27,7 @@ namespace XrdFileCache //---------------------------------------------------------------------------- //! Statistics of disk cache utilisation. //---------------------------------------------------------------------------- -class Stats : public XrdOucCacheStats +class Stats { public: //---------------------------------------------------------------------- @@ -41,18 +41,26 @@ public: long long m_BytesRam; //!< number of bytes served from RAM cache long long m_BytesMissed; //!< number of bytes served directly from XrdCl - inline void AddStat(Stats &Src) + inline void AddStats(Stats &Src) { - XrdOucCacheStats::Add(Src); - m_MutexXfc.Lock(); - m_BytesDisk += Src.m_BytesDisk; - m_BytesRam += Src.m_BytesRam; + + m_BytesDisk += Src.m_BytesDisk; + m_BytesRam += Src.m_BytesRam; m_BytesMissed += Src.m_BytesMissed; m_MutexXfc.UnLock(); } + Stats Clone() + { + Stats ret; + m_MutexXfc.Lock(); + ret = *this; + m_MutexXfc.UnLock(); + return ret; + } + private: XrdSysMutex m_MutexXfc; }; diff --git a/src/XrdFileCache/XrdFileCacheVRead.cc b/src/XrdFileCache/XrdFileCacheVRead.cc index ebb7e1f93de..b9d72e387ea 100644 --- a/src/XrdFileCache/XrdFileCacheVRead.cc +++ b/src/XrdFileCache/XrdFileCacheVRead.cc @@ -92,11 +92,13 @@ int File::ReadV(const XrdOucIOVec *readV, int n) return -1; } + Stats loc_stats; + int bytesRead = 0; - ReadVBlockListRAM blocks_to_process; + ReadVBlockListRAM blocks_to_process; std::vector blks_processed; - ReadVBlockListDisk blocks_on_disk; + ReadVBlockListDisk blocks_on_disk; std::vector chunkVec; DirectResponseHandler *direct_handler = 0; @@ -124,9 +126,14 @@ int File::ReadV(const XrdOucIOVec *readV, int n) { int dr = VReadFromDisk(readV, n, blocks_on_disk); if (dr < 0) + { bytesRead = dr; + } else + { bytesRead += dr; + loc_stats.m_BytesDisk += dr; + } } // read from cached blocks @@ -134,9 +141,14 @@ int File::ReadV(const XrdOucIOVec *readV, int n) { int br = VReadProcessBlocks(readV, n, blocks_to_process.bv, blks_processed); if (br < 0) + { bytesRead = br; + } else + { bytesRead += br; + loc_stats.m_BytesRam += br; + } } // check direct requests have arrived, get bytes read from read handle @@ -154,7 +166,7 @@ int File::ReadV(const XrdOucIOVec *readV, int n) for (std::vector::iterator i = chunkVec.begin(); i != chunkVec.end(); ++i) { bytesRead += i->size; - m_stats.m_BytesMissed += i->size; + loc_stats.m_BytesMissed += i->size; } } else @@ -183,6 +195,8 @@ int File::ReadV(const XrdOucIOVec *readV, int n) for (std::vector::iterator i = blks_processed.begin(); i != blks_processed.end(); ++i) delete i->arr; + m_stats.AddStats(loc_stats); + TRACEF(Dump, "VRead exit, total = " << bytesRead); return bytesRead; } @@ -292,10 +306,9 @@ int File::VReadFromDisk(const XrdOucIOVec *readV, int n, ReadVBlockListDisk& blo overlap(blockIdx, m_cfi.GetBufferSize(), readV[chunkIdx].offset, readV[chunkIdx].size, off, blk_off, size); int rs = m_output->Read(readV[chunkIdx].data + off, blockIdx*m_cfi.GetBufferSize() + blk_off - m_offset, size); - if (rs >=0) + if (rs >= 0) { bytes_read += rs; - m_stats.m_BytesDisk += rs; } else { @@ -358,8 +371,7 @@ int File::VReadProcessBlocks(const XrdOucIOVec *readV, int n, int block_idx = bi->block->m_offset/m_cfi.GetBufferSize(); overlap(block_idx, m_cfi.GetBufferSize(), readV[*chunkIt].offset, readV[*chunkIt].size, off, blk_off, size); memcpy(readV[*chunkIt].data + off, &(bi->block->m_buff[blk_off]), size); - bytes_read += size; - m_stats.m_BytesRam += size; + bytes_read += size; } } else