Merge pull request #1080 from rouault/vsicurl_lru_cache

/vsicurl/ and derived: implement a LRU cache for file properties and directory content listing
OSGeo · Nov 8, 2018 · c1985e7 · c1985e7
2 parents 0c16944 + d461b1b
commit c1985e7
Show file tree

Hide file tree

Showing 7 changed files with 395 additions and 340 deletions.
diff --git a/gdal/port/cpl_mem_cache.h b/gdal/port/cpl_mem_cache.h
@@ -172,6 +172,16 @@ class Cache {
     return cache_.find(k) != cache_.end();
   }
 
+  bool getOldestEntry(Key& kOut, Value& vOut) {
+    Guard g(lock_);
+    if( keys_.empty() ) {
+        return false;
+    }
+    kOut = keys_.back().key;
+    vOut = keys_.back().value;
+    return true;
+  }
+
   size_t getMaxSize() const { return maxSize_; }
   size_t getElasticity() const { return elasticity_; }
   size_t getMaxAllowedSize() const { return maxSize_ + elasticity_; }

diff --git a/gdal/port/cpl_vsil_az.cpp b/gdal/port/cpl_vsil_az.cpp
@@ -83,7 +83,7 @@ void VSICurlFilesystemHandler::AnalyseAzureFileList(
         return;
     CPLXMLNode* psEnumerationResults = CPLGetXMLNode(psTree, "=EnumerationResults");
 
-    std::vector< std::pair<CPLString, CachedFileProp> > aoProps;
+    std::vector< std::pair<CPLString, FileProp> > aoProps;
     // Count the number of occurrences of a path. Can be 1 or 2. 2 in the case
     // that both a filename and directory exist
     std::map<CPLString, int> aoNameCount;
@@ -115,7 +115,7 @@ void VSICurlFilesystemHandler::AnalyseAzureFileList(
                 {
                     bNonEmpty = true;
 
-                    CachedFileProp prop;
+                    FileProp prop;
                     prop.eExists = EXIST_YES;
                     prop.bHasComputedFileSize = true;
                     prop.fileSize = static_cast<GUIntBig>(
@@ -148,7 +148,7 @@ void VSICurlFilesystemHandler::AnalyseAzureFileList(
                     }
 
                     aoProps.push_back(
-                        std::pair<CPLString, CachedFileProp>
+                        std::pair<CPLString, FileProp>
                             (pszKey + osPrefix.size(), prop));
                     aoNameCount[pszKey + osPrefix.size()] ++;
                 }
@@ -166,15 +166,15 @@ void VSICurlFilesystemHandler::AnalyseAzureFileList(
                         osKey.resize(osKey.size()-1);
                     if( osKey.size() > osPrefix.size() )
                     {
-                        CachedFileProp prop;
+                        FileProp prop;
                         prop.eExists = EXIST_YES;
                         prop.bIsDirectory = true;
                         prop.bHasComputedFileSize = true;
                         prop.fileSize = 0;
                         prop.mTime = 0;
 
                         aoProps.push_back(
-                            std::pair<CPLString, CachedFileProp>
+                            std::pair<CPLString, FileProp>
                                 (osKey.c_str() + osPrefix.size(), prop));
                         aoNameCount[osKey.c_str() + osPrefix.size()] ++;
                     }
@@ -211,7 +211,7 @@ void VSICurlFilesystemHandler::AnalyseAzureFileList(
 #if DEBUG_VERBOSE
                 CPLDebug("AZURE", "Cache %s", osCachedFilename.c_str());
 #endif
-                *GetCachedFileProp(osCachedFilename) = aoProps[i].second;
+                SetCachedFileProp(osCachedFilename, aoProps[i].second);
             }
             osFileList.AddString( (aoProps[i].first + osSuffix).c_str() );
         }