Merge branch 'esri-pm' into esri

io/EptReader.cpp

@@ -470,7 +470,7 @@ void EptReader::ready(PointTableRef table)
     }
 
     // Ten million is a silly-large number for the number of tiles.
-    m_pool.reset(new ThreadPool(m_pool->numThreads(), 10000000));
+    m_pool.reset(new ThreadPool(m_pool->numThreads()));
     m_pointId = 0;
     m_tileCount = m_hierarchy->size();
 

pdal/util/ThreadPool.cpp

@@ -59,6 +59,8 @@ void ThreadPool::work()
             m_produceCv.notify_all();
 
             std::string err;
+task();
+/**
             try
             {
                 task();
@@ -71,7 +73,7 @@ void ThreadPool::work()
             {
                 err = "Unknown error";
             }
-
+**/
             lock.lock();
             --m_outstanding;
             if (err.size())

pdal/util/ThreadPool.hpp

@@ -34,6 +34,7 @@
 
 #pragma once
 
+#include <cassert>
 #include <condition_variable>
 #include <functional>
 #include <queue>
@@ -51,14 +52,11 @@ class PDAL_DLL ThreadPool
     // been enqueued to wait for an available worker thread, subsequent calls
     // to Pool::add will block until an enqueued task has been popped from the
     // queue.
-    ThreadPool(
-            std::size_t numThreads,
-            std::size_t queueSize = 1,
-            bool verbose = true)
-        : m_verbose(verbose)
-        , m_numThreads(std::max<std::size_t>(numThreads, 1))
-        , m_queueSize(std::max<std::size_t>(queueSize, 1))
+    ThreadPool(std::size_t numThreads, int64_t queueSize = -1, bool verbose = true) :
+        m_queueSize(queueSize) , m_numThreads(std::max<std::size_t>(numThreads, 1)),
+        m_verbose(verbose)
     {
+        assert(m_queueSize != 0);
         go();
     }
 
@@ -141,7 +139,7 @@ class PDAL_DLL ThreadPool
 
         m_produceCv.wait(lock, [this]()
         {
-            return m_tasks.size() < m_queueSize;
+            return m_queueSize < 0 || m_tasks.size() < m_queueSize;
         });
 
         m_tasks.emplace(task);
@@ -159,9 +157,9 @@ class PDAL_DLL ThreadPool
     // Worker thread function.  Wait for a task and run it.
     void work();
 
-    bool m_verbose;
-    std::size_t m_numThreads;
     std::size_t m_queueSize;
+    std::size_t m_numThreads;
+    bool m_verbose;
     std::vector<std::thread> m_threads;
     std::queue<std::function<void()>> m_tasks;
 

plugins/i3s/CMakeLists.txt

@@ -8,6 +8,7 @@ PDAL_ADD_PLUGIN(i3s_reader_libname reader i3s
         io/EsriUtil.cpp
         io/EsriReader.cpp
         io/Obb.cpp
+        io/PageManager.cpp
         ${PDAL_SRC_DIR}/private/MathUtils.cpp
     LINK_WITH
         ${WINSOCK_LIBRARY}
@@ -30,6 +31,7 @@ PDAL_ADD_PLUGIN(slpk_reader_libname reader slpk
         io/EsriUtil.cpp
         io/EsriReader.cpp
         io/Obb.cpp
+        io/PageManager.cpp
         ${PDAL_SRC_DIR}/private/MathUtils.cpp
     LINK_WITH
         ${WINSOCK_LIBRARY}

plugins/i3s/io/EsriReader.cpp

@@ -344,13 +344,15 @@ void EsriReader::ready(PointTableRef table)
     -texture data: <node-url>/textures/<texture-data-bundle-id>
     */
 
-    // Build the node list that will tell us which nodes overlap with bounds
-    std::string filename = "nodepages/0";
-    std::string s = fetchJson(filename);
-    NL::json initJson = i3s::parse(s,
-        "Invalid json in file '" + filename + "'.");
-    log()->get(LogLevel::Debug) << "Traversing metadata" << std::endl;
-    traverseTree(initJson, 0, 0, 0);
+    // Before version 2, page indexes were the first item on the page, rather than the
+    // actual page number, so we use the page size (m_nodeCap) as a factor from the page
+    // index to get the proper filename for a page.
+    int indexFactor = m_version >= Version("2.0") ? 1 : m_nodeCap;
+    i3s::FetchFunction fetch = std::bind(&EsriReader::fetchJson, this, std::placeholders::_1);
+    m_pageManager.reset(new PageManager(100, 4, indexFactor, fetch));
+    PagePtr p = m_pageManager->getPage(0);
+    traverseTree(p, 0);
+    m_pool->await();
 
     // If we're running in standard mode, queue up all the requests for data.
     // In streaming mode, queue up at most 4 to avoid having a ton of data
@@ -539,36 +541,32 @@ bool EsriReader::processPoint(PointRef& dst, const TileContents& tile)
 // the tree and test if it overlaps with the bounds created by user.
 // If it's a leaf node(the highest resolution) and it overlaps, add
 // it to the list of nodes to be pulled later.
-void EsriReader::traverseTree(NL::json page, int index, int depth,
-    int pageIndex)
+void EsriReader::traverseTree(PagePtr page, int node)
 {
+    int index = node % m_nodeCap;
+
     // find node information
-    int firstNode = page["nodes"][0]["resourceId"].get<int>();
-    int name = page["nodes"][index]["resourceId"].get<int>();
-    int firstChild = page["nodes"][index]["firstChild"].get<int>();
-    int cCount = page["nodes"][index]["childCount"].get<int>();
+    NL::json& j = *page;
+    int firstNode = j["nodes"][0]["resourceId"].get<int>();
+    int name = j["nodes"][index]["resourceId"].get<int>();
+    int firstChild = j["nodes"][index]["firstChild"].get<int>();
+    int cCount = j["nodes"][index]["childCount"].get<int>();
 
     // find density information
-    double area = page["nodes"][index][
+    double area = j["nodes"][index][
         m_version >= Version("2.0") ?
             "lodThreshold" :
             "effectiveArea" ].get<double>();
-    int pCount = page["nodes"][index][
+    int pCount = j["nodes"][index][
         m_version >= Version("2.0") ?
             "vertexCount" :
             "pointCount" ].get<int>();
 
     double density = pCount / area;
 
-    // update maximum node to stop reading files at the right time
-    if ((firstChild + cCount - 1) > m_maxNode)
-    {
-        m_maxNode = firstChild + cCount - 1;
-    }
-
     try
     {
-        Obb obb(page["nodes"][index]["obb"]);
+        Obb obb(j["nodes"][index]["obb"]);
         if (m_ecefTransform)
             obb.transform(*m_ecefTransform);
         if (m_args->obb.valid() && !obb.intersect(m_args->obb))
@@ -586,51 +584,33 @@ void EsriReader::traverseTree(NL::json page, int index, int depth,
     // if it's a child node and we're fetching full density, add leaf nodes
     if (m_args->max_density == -1 && m_args->min_density == -1 && cCount == 0)
     {
+        std::unique_lock<std::mutex> lock(m_mutex);
         m_nodes.push_back(name);
         return;
     }
-    else
+    if (density < m_args->max_density && density > m_args->min_density)
     {
-        if (density < m_args->max_density && density > m_args->min_density)
-        {
-            m_nodes.push_back(name);
-        }
-        // if we've already reached the last node, stop the process, otherwise
-        // increment depth and begin looking at child nodes
-        if (name == m_maxNode)
-            return;
+        std::unique_lock<std::mutex> lock(m_mutex);
+        m_nodes.push_back(name);
+    }
 
-        ++depth;
-        for (int i = 0; i < cCount; ++i)
-        {
-            if ((firstChild + i) > (firstNode + m_nodeCap - 1))
-            {
-                pageIndex =
-                    (m_version >= Version("2.0") ?
-                        (firstChild + i) / m_nodeCap :
-                        ((firstChild + i) / m_nodeCap) * m_nodeCap);
-
-                if (m_nodepages.find(pageIndex) != m_nodepages.end())
-                    page = m_nodepages[pageIndex];
-                else
-                {
-                    std::string filename = "nodepages/" +
-                        std::to_string(pageIndex);
-                    std::string s = fetchJson(filename);
-                    page = i3s::parse(s,
-                        "Invalid JSON in file '" + filename + "'.");
-                    m_nodepages[pageIndex] = page;
-                }
-            }
-            if (pageIndex != 0)
-                index =
-                    (m_version >= Version("2.0") ?
-                        (firstChild + i) % (m_nodeCap * pageIndex):
-                        (firstChild + i) % (pageIndex));
-            else
-                index = firstChild + i;
-            traverseTree(page, index, depth, pageIndex);
-        }
+    // if have no children, we're done this branch,.
+    if (cCount == 0)
+        return;
+
+    for (int i = 0; i < cCount; ++i)
+    {
+        int node = firstChild + i;
+        if (i == 0 || node % m_nodeCap == 0)
+            m_pageManager->fetchPage(node / m_nodeCap);
+    }
+
+    for (int i = 0; i < cCount; ++i)
+    {
+        node = firstChild + i;
+        if (i == 0 || node % m_nodeCap == 0)
+            page = m_pageManager->getPage(node / m_nodeCap);
+        m_pool->add([this, page, node](){traverseTree(page, node);});
     }
 }
 

plugins/i3s/io/EsriReader.hpp

@@ -49,6 +49,7 @@
 #include <pdal/util/Bounds.hpp>
 
 #include "EsriUtil.hpp"
+#include "PageManager.hpp"
 
 namespace pdal
 {
@@ -78,14 +79,13 @@ class PDAL_DLL EsriReader : public Reader, public Streamable
     std::unique_ptr<Args> m_args;
     NL::json m_info;
     int m_nodeCap;
-    int m_maxNode = 0;
     i3s::Version m_version;
     SpatialReference m_nativeSrs;
+    std::unique_ptr<i3s::PageManager> m_pageManager;
     std::unique_ptr<SrsTransform> m_ecefTransform;
     std::unique_ptr<ThreadPool> m_pool;
     std::vector<DimData> m_esriDims;
     size_t m_extraDimCount;
-    std::map<int, NL::json> m_nodepages;
     std::vector<int> m_nodes;
     size_t m_curNodeIdx;
     size_t m_tilesToProcess;
@@ -102,7 +102,7 @@ class PDAL_DLL EsriReader : public Reader, public Streamable
     virtual point_count_t read(PointViewPtr view, point_count_t count) override;
     virtual bool processOne(PointRef&) override;
     void createView(std::string localUrl, int nodeIndex,  PointView& view);
-    void traverseTree(NL::json page, int index, int depth, int pageIndex);
+    void traverseTree(i3s::PagePtr page, int node);
     void load(int nodeId);
     TileContents loadPath(const std::string& url);
     void checkTile(const TileContents& tile);

plugins/i3s/io/EsriUtil.hpp

@@ -98,7 +98,14 @@ class Version
     { return !(*this < other); }
     bool operator > (const Version& other)
     { return !(*this < other) && !(*this == other); }
+
+    friend std::ostream& operator<<(std::ostream& out, const Version & v);
 };
+inline std::ostream& operator<<(std::ostream& out, const Version & v)
+{
+    out << v.major << "." << v.minor << "." << v.patch;
+    return out;
+}
 
 class EsriError : public std::runtime_error
 {