Merge pull request #1082 from valhalla/kk_lz4

allow for lazy loading of raw height tiles

src/skadi/sample.cc

@@ -42,12 +42,35 @@ namespace {
           files.push_back(i->path().string());
     }//couldn't get data
     catch(...) {
-      LOG_WARN(root_dir + " has no elevation tiles");
+      LOG_WARN(root_dir + " currently has no elevation tiles");
       files.clear();
     }
     return files;
   }
 
+  std::string name_hgt(int16_t index) {
+    auto x = (index % 360) - 180;
+    auto y = (index / 360) - 90;
+
+    std::string name(y < 0 ? "/S" : "/N");
+    y = std::abs(y);
+    if(y < 10)
+      name.push_back('0');
+    name.append(std::to_string(y));
+    name.append(name);
+
+    name.append(x < 0 ? "W" : "E");
+    x = std::abs(x);
+    if(x < 100)
+      name.push_back('0');
+    if(x < 10)
+      name.push_back('0');
+    name.append(std::to_string(x));
+    name.append(".hgt");
+
+    return name;
+  }
+
   template<typename fmt_t>
   uint16_t is_hgt(const std::string& name, fmt_t& fmt) {
     boost::smatch m;
@@ -103,7 +126,11 @@ namespace valhalla {
 namespace skadi {
 
   sample::sample(const std::string& data_source):
-      mapped_cache(TILE_COUNT), unzipped(-1, std::vector<int16_t>(HGT_PIXELS)) {
+      mapped_cache(TILE_COUNT), unzipped_cache(-1, std::vector<int16_t>(HGT_PIXELS)), data_source(data_source) {
+    //messy but needed
+    while(this->data_source.size() && this->data_source.back() == '/')
+      this->data_source.pop_back();
+
     //check the directory for files that look like what we need
     auto files = get_files(data_source);
     for(const auto& f : files) {
@@ -126,31 +153,43 @@ namespace skadi {
     //bail if its out of bounds
     if(index >= TILE_COUNT)
       return nullptr;
-    //we have it raw or dont have it
-    const auto& mapped = mapped_cache[index];
-    if(mapped.first == format_t::RAW || mapped.second.get() == nullptr)
+
+    //if we dont have anything maybe its lazy loaded
+    auto& mapped = mapped_cache[index];
+    if(mapped.second.get() == nullptr) {
+      auto f = data_source + name_hgt(index);
+      auto size = file_size(f);
+      if(size != HGT_BYTES)
+        return nullptr;
+      mapped.first = format_t::RAW;
+      mapped.second.map(f, size, POSIX_MADV_SEQUENTIAL);
+    }
+
+    //we have it raw or we dont
+    if(mapped.first == format_t::RAW)
       return static_cast<const int16_t*>(static_cast<const void*>(mapped.second.get()));
-    //if we have it already give it back
-    if(unzipped.first == index)
-      return unzipped.second.data();
+
+    //if we have it already unzipped
+    if(unzipped_cache.first == index)
+      return unzipped_cache.second.data();
 
     //we have to unzip it
     try {
       if(mapped.first == format_t::LZ4HC)
-        lunzip(mapped.second, unzipped.second.data());
+        lunzip(mapped.second, unzipped_cache.second.data());
       else
-        gunzip(mapped.second, unzipped.second.data());
+        gunzip(mapped.second, unzipped_cache.second.data());
     }//failed to unzip
     catch(...) {
       LOG_WARN("Corrupt compressed elevation data");
-      unzipped.first = -1;
+      unzipped_cache.first = -1;
       return nullptr;
     }
 
     //update the simple cache
     //TODO: LRU
-    unzipped.first = index;
-    return unzipped.second.data();
+    unzipped_cache.first = index;
+    return unzipped_cache.second.data();
   }
 
   template <class coord_t>

test/data/samplegz/N40/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

test/data/samplelz/N00/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

test/data/samplelz/N40/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

test/sample.cc

@@ -76,17 +76,17 @@ void create_tile() {
   std::vector<int16_t> tile(3601 * 3601, 0);
   for (const auto& p : pixels)
     tile[p.first] = p.second;
-  std::ofstream file("test/data/sample/N40W077.hgt", std::ios::binary | std::ios::trunc);
+  std::ofstream file("test/data/sample/N40/N40W077.hgt", std::ios::binary | std::ios::trunc);
   file.write(static_cast<const char*>(static_cast<void*>(tile.data())), sizeof(int16_t) * tile.size());
 
   //write it again but this time gzipped
   auto gzipped = gzip(tile);
-  std::ofstream gzfile("test/data/samplegz/N40W077.hgt.gz", std::ios::binary | std::ios::trunc);
+  std::ofstream gzfile("test/data/samplegz/N40/N40W077.hgt.gz", std::ios::binary | std::ios::trunc);
   gzfile.write(static_cast<const char*>(static_cast<void*>(gzipped.data())), gzipped.size());
 
   //write it again but this time lzipped
   auto lzipped = lzip(tile);
-  std::ofstream lzfile("test/data/samplelz/N40W077.hgt.lz4", std::ios::binary | std::ios::trunc);
+  std::ofstream lzfile("test/data/samplelz/N40/N40W077.hgt.lz4", std::ios::binary | std::ios::trunc);
   lzfile.write(lzipped.data(), lzipped.size());
 }
 
@@ -157,6 +157,29 @@ void edges() {
     throw std::runtime_error("Wrong value at location");
 }
 
+void lazy_load() {
+  //make sure there is no data there
+  {
+    std::ofstream file("test/data/sample/N00/N00E000.hgt", std::ios::binary | std::ios::trunc);
+  }
+  skadi::sample s("test/data/sample");
+  if(s.get(std::make_pair(0.503915, 0.678783)) != s.get_no_data_value())
+    throw std::logic_error("Asked for point with no data should be no data value");
+
+  //put data there
+  {
+    std::vector<int16_t> tile(3601 * 3601, 0);
+    for (const auto& p : pixels)
+      tile[p.first] = p.second;
+    std::ofstream file("test/data/sample/N00/N00E000.hgt", std::ios::binary | std::ios::trunc);
+    file.write(static_cast<const char*>(static_cast<void*>(tile.data())), sizeof(int16_t) * tile.size());
+  }
+
+  //make sure there is now data there
+  if(std::fabs(490 - s.get(std::make_pair(1 - 0.503915, 0.678783))) > 1.0)
+    throw std::runtime_error("Wrong value at location: " + std::to_string(s.get(std::make_pair(1 - 0.503915, 0.678783))));
+}
+
 }
 
 int main() {
@@ -174,5 +197,7 @@ int main() {
 
   suite.test(TEST_CASE(getlz));
 
+  suite.test(TEST_CASE(lazy_load));
+
   return suite.tear_down();
 }

valhalla/midgard/sequence.h

@@ -111,6 +111,10 @@ class mem_map {
     return count;
   }
 
+  const std::string& name() const {
+    return file_name;
+  }
+
  protected:
 
   void* ptr;

valhalla/skadi/sample.h

@@ -55,13 +55,26 @@ namespace valhalla {
        */
       const int16_t* source(uint16_t index) const;
 
-      //using memory maps
+      /**
+       * maps a new source, used at start up and called periodically
+       * for lazily loaded sources
+       *
+       * @param  index  the index of the data tile being mapped
+       * @param  format the format of the data tile being mapped
+       * @param  file   the file name of the data tile being mapped
+       * @return none
+       */
       enum class format_t{ UNKNOWN = 0, GZIP = 1, LZ4HC = 2, RAW = 3 };
-      std::vector<std::pair<format_t, midgard::mem_map<char> > > mapped_cache;
+      void map(uint16_t index, format_t format, const std::string& file);
+
+      //using memory maps
+      mutable std::vector<std::pair<format_t, midgard::mem_map<char> > > mapped_cache;
 
       //TODO: make an LRU
       using unzipped_t = std::pair<int16_t, std::vector<int16_t> >;
-      mutable unzipped_t unzipped;
+      mutable unzipped_t unzipped_cache;
+
+      std::string data_source;
     };
 
   }

valhalla/valhalla.h.in

@@ -3,7 +3,7 @@
 
 #define VALHALLA_VERSION_MAJOR 2
 #define VALHALLA_VERSION_MINOR 4
-#define VALHALLA_VERSION_PATCH 5
+#define VALHALLA_VERSION_PATCH 6
 
 //whether the API was configured with http service support or not
 @HAVE_HTTP_DEFINE@