From 6210d99eed326d59f224ce563a40d71fcaf860a1 Mon Sep 17 00:00:00 2001 From: kylemann16 Date: Thu, 27 Sep 2018 17:31:16 -0500 Subject: [PATCH] changed lod to depth of node tree --- doc/stages/readers.i3s.rst | 7 +-- doc/stages/readers.slpk.rst | 6 +-- plugins/i3s/io/EsriReader.cpp | 78 ++++++++++++++++++++++++++++-- plugins/i3s/io/EsriReader.hpp | 8 +-- plugins/i3s/io/I3SReader.cpp | 60 +---------------------- plugins/i3s/io/I3SReader.hpp | 6 +-- plugins/i3s/io/SlpkReader.cpp | 60 +++-------------------- plugins/i3s/io/SlpkReader.hpp | 3 +- plugins/i3s/lepcc/CMakeLists.txt | 8 +-- plugins/i3s/test/i3sReaderTest.cpp | 26 +--------- 10 files changed, 106 insertions(+), 156 deletions(-) diff --git a/doc/stages/readers.i3s.rst b/doc/stages/readers.i3s.rst index 7d91cf6977..9df71566d3 100644 --- a/doc/stages/readers.i3s.rst +++ b/doc/stages/readers.i3s.rst @@ -55,8 +55,9 @@ dimensions Example: ``--readers.i3s.dimensions="returns, rgb"`` -lod - This is the density of the points in the nodes that will be selected during the read. A density of 0 will select any nodes with a density calculated to be between 0 and 0.5. A higher density means the same extents will be looked at, but more points within thos bounds will be viewed. Higher resolution means more file fetches, so a lower resolution will most likely be faster. This number may change between files. - Example: ``--readers.i3s.lod=2`` +depth + The depth refers to the depth of the node tree that the user would like to see data from. This essentially alters the viewed resolution. This will select the highest resolution closest to the depth selected. 0 represents the 0th depth of the tree, and the minimum resolution of the data. Selecting nothing here will output the highest resolution in every node subtree. + + Example: ``--readers.i3s.depth=2`` .. _Indexed 3d Scene Layer (I3S): https://github.com/Esri/i3s-spec/blob/master/format/Indexed%203d%20Scene%20Layer%20Format%20Specification.md diff --git a/doc/stages/readers.slpk.rst b/doc/stages/readers.slpk.rst index 133a0e0bff..a88cc875cb 100644 --- a/doc/stages/readers.slpk.rst +++ b/doc/stages/readers.slpk.rst @@ -53,9 +53,9 @@ dimensions Example: ``--readers.slpk.dimensions="rgb, intensity"`` -lod - This is the density of the points in the nodes that will be selected during the read. A density of 0 will select any nodes with a density calculated to be between 0 and 0.5. A higher density means the same extents will be looked at, but more points within thos bounds will be viewed. This number may change between files. - Example: ``--readers.slpk.lod=2`` +depth + The depth refers to the depth of the node tree that the user would like to see data from. This essentially alters the viewed resolution. This will select the highest resolution closest to the depth selected. 0 represents the 0th depth of the tree, and the minimum resolution of the data. Selecting nothing here will output the highest resolution in every node subtree. + Example: ``--readers.slpk.depth=2`` .. _Scene Layer Packages (SLPK): https://github.com/Esri/i3s-spec/blob/master/format/Indexed%203d%20Scene%20Layer%20Format%20Specification.md#_8_1 .. _I3S: https://pdal.io/readers.i3s.html diff --git a/plugins/i3s/io/EsriReader.cpp b/plugins/i3s/io/EsriReader.cpp index 80c3a29426..3bce2c7504 100644 --- a/plugins/i3s/io/EsriReader.cpp +++ b/plugins/i3s/io/EsriReader.cpp @@ -61,7 +61,7 @@ void EsriReader::addArgs(ProgramArgs& args) args.add("bounds", "Bounds of the point cloud", m_args.bounds); args.add("threads", "Number of threads to be used." , m_args.threads); args.add("dimensions", "Dimensions to be used in pulls", m_args.dimensions); - args.add("lod", "Point density of nodes selected", m_args.lod, -1.00); + args.add("depth", "Point density of nodes selected", m_args.depth, -1.00); } @@ -230,7 +230,7 @@ void EsriReader::ready(PointTableRef) point_count_t EsriReader::read(PointViewPtr view, point_count_t count) { /* - -3Dscenelayerinfo: URL Pattern + -3Dscenelayerinfo: -node index document: /nodepages/ -shared resources: /shared/ -feature data: /features/ @@ -240,9 +240,10 @@ point_count_t EsriReader::read(PointViewPtr view, point_count_t count) //Build the node list that will tell us which nodes overlap with bounds std::vector nodes; - buildNodeList(nodes, 0); + const Json::Value initJson = fetchJson(m_filename + "/nodepages/0"); + traverseTree(initJson, 0, nodes, 0, 0); - //Create view with overlapping leaf nodes + //Create view with overlapping nodes at desired depth //Will create a thread pool on the createview class and iterate //through the node list for the nodes to be pulled. log()->get(LogLevel::Debug) << "Fetching binaries" << std::endl; @@ -263,6 +264,74 @@ point_count_t EsriReader::read(PointViewPtr view, point_count_t count) return view->size(); } +//Traverse tree through nodepages. Create a nodebox for each node in +//the tree and test if it overlaps with the bounds created by user. +//If it's a leaf node(the highest resolution) and it overlaps, add +//it to the list of nodes to be pulled later. +void EsriReader::traverseTree(Json::Value page, int index, + std::vector& nodes, int depth, int pageIndex) +{ + + //find node information + int name = page["nodes"][index]["resourceId"].asInt(); + int firstChild = page["nodes"][index]["firstChild"].asInt(); + int cCount = page["nodes"][index]["childCount"].asInt(); + + //update maximum node to stop reading files at the right time + if ((firstChild + cCount - 1) > m_maxNode) + { + m_maxNode = firstChild + cCount - 1; + } + + //create box from OBB information and check for overlaps + BOX3D nodeBox = parseBox(page["nodes"][index]); + bool overlap = m_bounds.overlaps(nodeBox); + + //if it doesn't overlap, then none of the nodes in this subtree will + if(!overlap) + return; + //if it's a child node and the depth hasn't been reached, add it + if (cCount == 0) + { + nodes.push_back(name); + return; + } + else if (depth == m_args.depth) + { + nodes.push_back(name); + return; + } + else + { + //if we've already reached the last node stop, other wise increment + //depth and begin looking at child nodes + if (name == m_maxNode) + return; + ++depth; + for (int i = 0; i < cCount; ++i) + { + if ((firstChild + i)/m_nodeCap != pageIndex) + { + pageIndex = (firstChild + i)/m_nodeCap; + std::string output; + + if(m_nodepages.find(pageIndex) != m_nodepages.end()) + page = m_nodepages[pageIndex]; + else + { + page = fetchJson(m_filename + "/nodepages/" + + std::to_string(pageIndex)); + m_nodepages[pageIndex] = page; + } + } + if(pageIndex != 0) + index = (firstChild + i) % (m_nodeCap*pageIndex); + else + index = firstChild + i; + traverseTree(page, index, nodes, depth, pageIndex); + } + } +} // Create the BOX3D for the node. This will be used for testing overlap. BOX3D EsriReader::parseBox(Json::Value base) { @@ -377,6 +446,7 @@ void EsriReader::createView(std::string localUrl, PointView& view) const std::string attrUrl = localUrl + "/attributes/"; + //the extensions seen in this part correspond with slpk for (const auto& dimEntry : m_dimMap) { const Dimension::Id dimId(dimEntry.first); diff --git a/plugins/i3s/io/EsriReader.hpp b/plugins/i3s/io/EsriReader.hpp index d39879abab..1c2b94499f 100644 --- a/plugins/i3s/io/EsriReader.hpp +++ b/plugins/i3s/io/EsriReader.hpp @@ -95,10 +95,9 @@ class PDAL_DLL EsriReader : public Reader protected: virtual void initInfo() = 0; - virtual void buildNodeList(std::vector& nodes, int pageIndex) = 0; virtual std::vector fetchBinary(std::string url, std::string attNum, std::string ext) const = 0; - + virtual Json::Value fetchJson(std::string) = 0; std::unique_ptr m_stream; @@ -109,7 +108,7 @@ class PDAL_DLL EsriReader : public Reader Bounds bounds; uint16_t threads = 8; std::vector dimensions; - double lod = -1; + double depth; }; EsriArgs m_args; @@ -143,6 +142,7 @@ class PDAL_DLL EsriReader : public Reader std::string name; }; std::map m_dimMap; + std::map m_nodepages; virtual void addArgs(ProgramArgs& args) override; @@ -153,6 +153,8 @@ class PDAL_DLL EsriReader : public Reader virtual void done(PointTableRef table) override; void createView(std::string localUrl, PointView& view); BOX3D parseBox(Json::Value base); + void traverseTree(Json::Value page, int index, + std::vector& nodes, int depth, int pageIndex); }; diff --git a/plugins/i3s/io/I3SReader.cpp b/plugins/i3s/io/I3SReader.cpp index 900f79981c..c855d2ee6d 100644 --- a/plugins/i3s/io/I3SReader.cpp +++ b/plugins/i3s/io/I3SReader.cpp @@ -68,65 +68,9 @@ void I3SReader::initInfo() m_filename += "/layers/0"; } -//Traverse tree through nodepages. Create a nodebox for each node in -//the tree and test if it overlaps with the bounds created by user. -//If it's a leaf node(the highest resolution) and it overlaps, add -//it to the list of nodes to be pulled later. -void I3SReader::buildNodeList(std::vector& nodes, int pageIndex) +Json::Value I3SReader::fetchJson(std::string filepath) { - std::string nodeUrl = m_filename + "/nodepages/" - + std::to_string(pageIndex); - - const Json::Value nodeIndexJson = parse(m_arbiter->get(nodeUrl)); - - - if(nodeIndexJson.empty() || !nodeIndexJson.isMember("nodes")) - throwError(std::string("Could not find node information")); - - int pageSize = nodeIndexJson["nodes"].size(); - int initialNode = nodeIndexJson["nodes"][0]["resourceId"].asInt(); - - for (int i = 0; i < pageSize; i++) - { - BOX3D nodeBox = parseBox(nodeIndexJson["nodes"][i]); - int cCount = nodeIndexJson["nodes"][i]["childCount"].asInt(); - - //density calculated as (number of points in node) / (lod threshold) - int pCount = nodeIndexJson["nodes"][i]["vertexCount"].asInt(); - double lodThreshold = - nodeIndexJson["nodes"][i]["lodThreshold"].asDouble(); - double density = (double)pCount / lodThreshold; - bool overlap = m_bounds.overlaps(nodeBox); - - //if density is within desired lod and the bounds overlap with node - //lod is default at -1, meaning no user input. This will grab only - //leaf nodes, or the highest resolution. - if (m_args.lod == -1 && overlap && cCount == 0) - { - int name = nodeIndexJson["nodes"][i]["resourceId"].asInt(); - nodes.push_back(name); - } - //0.5 represents a large enough gap to find the related nodes - //while also separating from different resolution sets - else if (density > m_args.lod && - density < (m_args.lod + 0.5) && - overlap) - { - int name = nodeIndexJson["nodes"][i]["resourceId"].asInt(); - nodes.push_back(name); - } - - //keeps track of largest node so recursive loop knows when to stop - if ((nodeIndexJson["nodes"][i]["firstChild"].asInt() + - cCount - 1) > m_maxNode) - { - m_maxNode = nodeIndexJson["nodes"][i]["firstChild"].asInt() + - cCount - 1; - } - else if (initialNode + i == m_maxNode) - return; - } - buildNodeList(nodes, ++pageIndex); + return parse(m_arbiter->get(filepath)); } std::vector I3SReader::fetchBinary(std::string url, diff --git a/plugins/i3s/io/I3SReader.hpp b/plugins/i3s/io/I3SReader.hpp index 4dbaaba353..30cb7fbdaf 100644 --- a/plugins/i3s/io/I3SReader.hpp +++ b/plugins/i3s/io/I3SReader.hpp @@ -37,7 +37,7 @@ namespace pdal { - + class PDAL_DLL I3SReader : public EsriReader { public: @@ -45,8 +45,8 @@ class PDAL_DLL I3SReader : public EsriReader protected: virtual void initInfo() override; - virtual void buildNodeList(std::vector& nodes, int pageIndex) override; virtual std::vector fetchBinary(std::string url, std::string attNum, std::string ext) const override; + virtual Json::Value fetchJson(std::string) override; }; -} \ No newline at end of file +} diff --git a/plugins/i3s/io/SlpkReader.cpp b/plugins/i3s/io/SlpkReader.cpp index fe5c475061..bdc1b4d7fb 100644 --- a/plugins/i3s/io/SlpkReader.cpp +++ b/plugins/i3s/io/SlpkReader.cpp @@ -99,67 +99,21 @@ void SlpkReader::initInfo() } -//Traverse tree through nodepages. Create a nodebox for each node in -//the tree and test if it overlaps with the bounds created by user. -//If it's a leaf node(the highest resolution) and it overlaps, add -//it to the list of nodes to be pulled later. -void SlpkReader::buildNodeList(std::vector& nodes, int pageIndex) -{ - std::string nodeUrl = m_filename + "/nodepages/" - + std::to_string(pageIndex); - - std::string ext = ".json.gz"; - - if(!FileUtils::fileExists(nodeUrl + ext)) - { - return; - } +//fetch json because they can't make it the same type +Json::Value SlpkReader::fetchJson(std::string filepath) +{ std::string output; - auto compressed = m_arbiter->get(nodeUrl+ext); + auto compressed = m_arbiter->get(filepath + ".json.gz"); m_decomp.decompress( output, compressed.data(), compressed.size()); - const Json::Value nodeIndexJson = parse(output); - - if(nodeIndexJson.empty() || !nodeIndexJson.isMember("nodes")) - throwError(std::string("Could not find node information")); - - int pageSize = nodeIndexJson["nodes"].size(); - int initialNode = nodeIndexJson["nodes"][0]["resourceId"].asInt(); - - for (int i = 0; i < pageSize; i++) - { - BOX3D nodeBox = parseBox(nodeIndexJson["nodes"][i]); - int cCount = nodeIndexJson["nodes"][i]["childCount"].asInt(); - - //density calculated as (number of points in node) / (lod threshold) - int pCount = nodeIndexJson["nodes"][i]["vertexCount"].asInt(); - double lodThreshold = - nodeIndexJson["nodes"][i]["lodThreshold"].asDouble(); - double density = (double)pCount / lodThreshold; - bool overlap = m_bounds.overlaps(nodeBox); - - //if density is within desired lod and the bounds overlap with node - //lod is default at -1, meaning no user input. This will grab only - //leaf nodes, or the highest resolution. - if(m_args.lod == -1 && overlap && cCount == 0) - { - int name = nodeIndexJson["nodes"][i]["resourceId"].asInt(); - nodes.push_back(name); - } - else if (density > m_args.lod && - density < (m_args.lod + 0.5) && - overlap) - { - int name = nodeIndexJson["nodes"][i]["resourceId"].asInt(); - nodes.push_back(name); - } - } - buildNodeList(nodes, ++pageIndex); + return parse(output); + } +//fetch data using arbiter to get a char vector std::vector SlpkReader::fetchBinary(std::string url, std::string attNum, std::string ext) const { diff --git a/plugins/i3s/io/SlpkReader.hpp b/plugins/i3s/io/SlpkReader.hpp index a6a4302fd0..8006b94f18 100644 --- a/plugins/i3s/io/SlpkReader.hpp +++ b/plugins/i3s/io/SlpkReader.hpp @@ -44,9 +44,10 @@ class PDAL_DLL SlpkReader : public EsriReader protected: virtual void initInfo() override; - virtual void buildNodeList(std::vector& nodes, int pageIndex) override; virtual std::vector fetchBinary(std::string url, std::string attNum, std::string ext) const override; + virtual Json::Value fetchJson(std::string) override; + }; } // namespace pdal diff --git a/plugins/i3s/lepcc/CMakeLists.txt b/plugins/i3s/lepcc/CMakeLists.txt index 365812a75b..af7961d097 100644 --- a/plugins/i3s/lepcc/CMakeLists.txt +++ b/plugins/i3s/lepcc/CMakeLists.txt @@ -23,9 +23,9 @@ target_include_directories(lepcc PRIVATE ${LEPCC_INCLUDE_DIR}) -add_executable(lepcctest ${TEST_SRCS}) -target_include_directories(lepcctest PRIVATE - ${LEPCC_INCLUDE_DIR}) -target_link_libraries(lepcctest lepcc) +#add_executable(lepcctest ${TEST_SRCS}) +#target_include_directories(lepcctest PRIVATE +# ${LEPCC_INCLUDE_DIR}) +#target_link_libraries(lepcctest lepcc) add_compile_options(-fPIC) diff --git a/plugins/i3s/test/i3sReaderTest.cpp b/plugins/i3s/test/i3sReaderTest.cpp index fcb53622fe..700667350a 100644 --- a/plugins/i3s/test/i3sReaderTest.cpp +++ b/plugins/i3s/test/i3sReaderTest.cpp @@ -52,7 +52,7 @@ TEST(i3sReaderTest, bounds_and_lod_test) i3s_options.add("filename", "i3s://https://tiles.arcgis.com/tiles/8cv2FuXuWSfF0nbL/arcgis/rest/services/AUTZEN_LiDAR/SceneServer"); i3s_options.add("threads", 64); i3s_options.add("bounds", "([-123.077,-123.063],[44.053, 44.060], [130, 175])"); - i3s_options.add("lod", 0); + i3s_options.add("depth", 3); @@ -73,7 +73,7 @@ TEST(i3sReaderTest, bounds_and_lod_test) Options options2; options2.add("filename", "i3s://https://tiles.arcgis.com/tiles/8cv2FuXuWSfF0nbL/arcgis/rest/services/AUTZEN_LiDAR/SceneServer"); options2.add("threads", 64); - options2.add("lod", 0); + options2.add("depth", 3); I3SReader reader2; @@ -112,27 +112,5 @@ TEST(i3sReaderTest, bounds_and_lod_test) arbiter.reset(new arbiter::Arbiter(config)); std::string url = "https://tiles.arcgis.com/tiles/8cv2FuXuWSfF0nbL/arcgis/rest/services/AUTZEN_LiDAR/SceneServer/layers/0/nodepages/"; - - - - uint64_t finalCount = 0; - for (int i = 0; i < 31; ++i) - { - const Json::Value nodepage = parse(arbiter->get( - url + std::to_string(i))); - for(const Json::Value& node : nodepage["nodes"]) - { - double lodThreshold = node["lodThreshold"].asDouble(); - uint64_t pCount = node["vertexCount"].asUInt64(); - double density = (double) pCount / lodThreshold; - if (density > 0 && - density < 0.5) - { - finalCount += pCount; - } - } - } - EXPECT_EQ(view2->size(), finalCount); - }