Keep only necessary elements in file purge map during FillFileMapRecu…

…rse.
xrootd · Mar 4, 2015 · 03dc632 · 03dc632
1 parent e1c38bc
commit 03dc632
Show file tree

Hide file tree

Showing 2 changed files with 66 additions and 12 deletions.
diff --git a/src/XrdFileCache/XrdFileCacheFactory.cc b/src/XrdFileCache/XrdFileCacheFactory.cc
@@ -371,9 +371,49 @@ bool Factory::ConfigParameters(std::string part, XrdOucStream& config )
 }
 
 //______________________________________________________________________________
+//namespace {
 
+class FPurgeState {
+public:
+   struct FS {
+      std::string path;
+      int nBlck;
 
-void FillFileMapRecurse( XrdOssDF* iOssDF, const std::string& path, std::multimap<time_t, std::string>& fcmap)
+      FS(const char* p, int n) : path(p), nBlck(n) {}
+   };
+
+   typedef std::multimap<time_t, FS> map_t;
+   typedef map_t::iterator map_i;
+
+   FPurgeState(long long iNBlckReq) : nBlckReq(iNBlckReq), nBlckAccum(0) {}
+
+   map_t fmap;
+
+   void checkFile (time_t iTime, const char* iPath,  int iNBlck) 
+   {
+      if ( (nBlckAccum <  nBlckReq ) || (iTime < fmap.rbegin()->first) ) {
+         fmap.insert(std::pair<time_t, FS> (iTime, FS(iPath, iNBlck)));
+         nBlckAccum += iNBlck;
+
+         // remove newest files from map if necessary
+         while (nBlckAccum > nBlckReq) {
+            time_t nt = fmap.begin()->first;
+            std::pair<map_i, map_i> ret = fmap.equal_range(nt); 
+            for (map_i it2 = ret.first; it2 != ret.second; ++it2)
+               nBlckAccum -= it2->second.nBlck;
+         }
+      }
+   }
+
+   private:
+      long long nBlckReq;
+      long long nBlckAccum;
+};
+
+
+//}
+
+void FillFileMapRecurse( XrdOssDF* iOssDF, const std::string& path, FPurgeState& purgeState)
 {
    char buff[256];
    XrdOucEnv env;
@@ -407,7 +447,7 @@ void FillFileMapRecurse( XrdOssDF* iOssDF, const std::string& path, std::multima
             if (cinfo.GetLatestDetachTime(accessTime, fh))
             {
                log->Debug(XrdCl::AppMsg, "FillFileMapRecurse() checking %s accessTime %d ", buff, (int)accessTime);
-               fcmap.insert(std::pair<time_t, std::string> (accessTime, buff));
+               purgeState.checkFile(accessTime, buff, cinfo.GetNDownloadedBlocks());
             }
             else
             {
@@ -416,7 +456,7 @@ void FillFileMapRecurse( XrdOssDF* iOssDF, const std::string& path, std::multima
          }
          else if ( dh->Opendir(np.c_str(), env)  >= 0 )
          {
-            FillFileMapRecurse(dh, np, fcmap);
+            FillFileMapRecurse(dh, np, purgeState);
          }
 
          delete dh; dh = 0;
@@ -459,22 +499,21 @@ void Factory::CacheDirCleanup()
 
       if (bytesToRemove > 0)
       {
-         typedef std::multimap<time_t, std::string> fcmap_t;
-         fcmap_t fcmap;
-         // make a sorted map of file patch by access time
+        // make a sorted map of file patch by access time
          XrdOssDF* dh = oss->newDir(m_configuration.m_username.c_str());
          if (dh->Opendir(m_configuration.m_cache_dir.c_str(), env) >= 0)
          {
-            FillFileMapRecurse(dh, m_configuration.m_cache_dir, fcmap);
+            long long nReq = (bytesToRemove*1.4)/m_configuration.m_bufferSize; // check more that required
+            FPurgeState purgeState(nReq);
+            FillFileMapRecurse(dh, m_configuration.m_cache_dir, purgeState);
 
             // loop over map and remove files with highest value of access time
-            for (fcmap_t::iterator it = fcmap.begin(); it != fcmap.end(); ++it)
+            for (FPurgeState::map_i it = purgeState.fmap.begin(); it != purgeState.fmap.end(); ++it)
             {
-               std::pair<fcmap_t::iterator, fcmap_t::iterator> ret;
-               ret = fcmap.equal_range(it->first); 
-               for (fcmap_t::iterator it2 = ret.first; it2 != ret.second; ++it2)
+               std::pair<FPurgeState::map_i, FPurgeState::map_i> ret = purgeState.fmap.equal_range(it->first); 
+               for (FPurgeState::map_i it2 = ret.first; it2 != ret.second; ++it2)
                {
-                  std::string path = it2->second;
+                  std::string path = it2->second.path;
                   // remove info file
                   if (oss->Stat(path.c_str(), &fstat) == XrdOssOK)
                   {

diff --git a/src/XrdFileCache/XrdFileCacheInfo.hh b/src/XrdFileCache/XrdFileCacheInfo.hh
@@ -131,6 +131,11 @@ namespace XrdFileCache
          //---------------------------------------------------------------------
          bool IsComplete() const;
 
+         //---------------------------------------------------------------------
+         //! Get number of downloaded blocks
+         //---------------------------------------------------------------------
+         int GetNDownloadedBlocks() const;
+
          //---------------------------------------------------------------------
          //! Update complete status
          //---------------------------------------------------------------------
@@ -171,6 +176,16 @@ namespace XrdFileCache
       return (m_buff_fetched[cn] & cfiBIT(off)) == cfiBIT(off);
    }
 
+
+   inline int Info::GetNDownloadedBlocks() const
+   {
+      int cntd = 0;
+      for (int i = 0; i < m_sizeInBits; ++i)
+         if (TestBit(i)) cntd++;
+
+      return cntd;
+   }
+
    inline int Info::GetSizeInBytes() const
    {
       return ((m_sizeInBits -1)/8 + 1);