Skip to content
Permalink
Browse files

Merge branch 'master' of https://github.com/qgis/QGIS into hana-provider

  • Loading branch information
stefanuhrig committed Jan 7, 2021
2 parents 1558287 + 1181c36 commit cf5180a05195f088532a1f44dc9fb7ffc8259ac6
Showing with 2,816 additions and 1,302 deletions.
  1. +2 −1 debian/control.in
  2. +1 −0 doc/CONTRIBUTORS
  3. +26 −1 external/untwine/bu/BuPyramid.cpp
  4. +1 −0 external/untwine/bu/BuTypes.hpp
  5. +50 −6 external/untwine/bu/Processor.cpp
  6. +2 −1 external/untwine/bu/Processor.hpp
  7. +23 −3 external/untwine/bu/PyramidManager.cpp
  8. +4 −1 external/untwine/bu/PyramidManager.hpp
  9. +67 −0 external/untwine/bu/Stats.cpp
  10. +181 −0 external/untwine/bu/Stats.hpp
  11. +1 −0 external/untwine/epf/Grid.cpp
  12. +3 −0 external/untwine/epf/Reprocessor.cpp
  13. +1 −0 external/untwine/untwine/Common.hpp
  14. +3 −1 external/untwine/untwine/FileDimInfo.hpp
  15. +13 −7 external/untwine/untwine/ProgressWriter.cpp
  16. +13 −4 external/untwine/untwine/Untwine.cpp
  17. +23 −8 external/untwine/untwine/VoxelKey.hpp
  18. +1 −0 python/core/auto_additions/qgsabstractdatabaseproviderconnection.py
  19. +3 −3 python/core/auto_generated/effects/qgsblureffect.sip.in
  20. +3 −3 python/core/auto_generated/effects/qgscoloreffect.sip.in
  21. +3 −3 python/core/auto_generated/effects/qgseffectstack.sip.in
  22. +4 −4 python/core/auto_generated/effects/qgsgloweffect.sip.in
  23. +5 −5 python/core/auto_generated/effects/qgspainteffect.sip.in
  24. +2 −2 python/core/auto_generated/effects/qgspainteffectregistry.sip.in
  25. +4 −4 python/core/auto_generated/effects/qgsshadoweffect.sip.in
  26. +3 −3 python/core/auto_generated/effects/qgstransformeffect.sip.in
  27. +1 −1 python/core/auto_generated/labeling/qgsrulebasedlabeling.sip.in
  28. +3 −3 python/core/auto_generated/labeling/qgsvectorlayerlabeling.sip.in
  29. +3 −1 python/core/auto_generated/processing/qgsprocessingparameters.sip.in
  30. +15 −12 python/core/auto_generated/qgscolorramp.sip.in
  31. +2 −0 python/core/auto_generated/qgsinterval.sip.in
  32. +54 −12 python/core/auto_generated/qgsrenderchecker.sip.in
  33. +86 −0 python/core/auto_generated/qgsrunprocess.sip.in
  34. +24 −0 python/core/auto_generated/qgstemporalutils.sip.in
  35. +1 −1 python/core/auto_generated/raster/qgshillshaderenderer.sip.in
  36. +1 −1 python/core/auto_generated/raster/qgsmultibandcolorrenderer.sip.in
  37. +1 −1 python/core/auto_generated/raster/qgspalettedrasterrenderer.sip.in
  38. +1 −1 python/core/auto_generated/raster/qgsrasterlayer.sip.in
  39. +1 −1 python/core/auto_generated/raster/qgsrasterrenderer.sip.in
  40. +1 −1 python/core/auto_generated/raster/qgssinglebandgrayrenderer.sip.in
  41. +1 −1 python/core/auto_generated/raster/qgssinglebandpseudocolorrenderer.sip.in
  42. +2 −2 python/core/auto_generated/symbology/qgsarrowsymbollayer.sip.in
  43. +2 −2 python/core/auto_generated/symbology/qgscategorizedsymbolrenderer.sip.in
  44. +7 −4 python/core/auto_generated/symbology/qgsellipsesymbollayer.sip.in
  45. +25 −25 python/core/auto_generated/symbology/qgsfillsymbollayer.sip.in
  46. +5 −2 python/core/auto_generated/symbology/qgsgeometrygeneratorsymbollayer.sip.in
  47. +1 −1 python/core/auto_generated/symbology/qgsgraduatedsymbolrenderer.sip.in
  48. +9 −9 python/core/auto_generated/symbology/qgslinesymbollayer.sip.in
  49. +19 −16 python/core/auto_generated/symbology/qgsmarkersymbollayer.sip.in
  50. +2 −2 python/core/auto_generated/symbology/qgsmasksymbollayer.sip.in
  51. +1 −1 python/core/auto_generated/symbology/qgspointdistancerenderer.sip.in
  52. +2 −2 python/core/auto_generated/symbology/qgsrenderer.sip.in
  53. +1 −1 python/core/auto_generated/symbology/qgsrendererrange.sip.in
  54. +5 −2 python/core/auto_generated/symbology/qgsrulebasedrenderer.sip.in
  55. +1 −1 python/core/auto_generated/symbology/qgssinglesymbolrenderer.sip.in
  56. +71 −31 python/core/auto_generated/symbology/qgssvgcache.sip.in
  57. +4 −4 python/core/auto_generated/symbology/qgssymbol.sip.in
  58. +8 −5 python/core/auto_generated/symbology/qgssymbollayer.sip.in
  59. +6 −6 python/core/auto_generated/symbology/qgssymbollayerregistry.sip.in
  60. +13 −7 python/core/auto_generated/symbology/qgssymbollayerutils.sip.in
  61. +6 −3 python/core/auto_generated/symbology/qgsvectorfieldsymbollayer.sip.in
  62. +1 −1 python/core/auto_generated/vector/qgsvectorlayer.sip.in
  63. +19 −0 python/gui/auto_generated/qgsmapcanvas.sip.in
  64. +22 −17 python/plugins/MetaSearch/dialogs/maindialog.py
  65. +26 −3 python/plugins/processing/algs/otb/OtbAlgorithm.py
  66. +3 −8 python/plugins/processing/gui/BatchPanel.py
  67. +4 −3 python/plugins/processing/gui/ParametersPanel.py
  68. +1 −1 python/plugins/processing/modeler/ModelerDialog.py
  69. +1 −1 python/server/auto_generated/qgsserverprojectutils.sip.in
  70. +5 −1 scripts/sipify_all.sh
  71. +7 −4 src/app/locator/qgsinbuiltlocatorfilters.cpp
  72. +14 −2 src/app/qgisapp.cpp
  73. +37 −21 src/app/qgsapplayertreeviewmenuprovider.cpp
  74. +7 −3 src/app/qgsidentifyresultsdialog.cpp
  75. +2 −2 src/app/qgsmapcanvasdockwidget.cpp
  76. +2 −2 src/core/annotations/qgsannotation.cpp
  77. +6 −6 src/core/effects/qgsblureffect.cpp
  78. +3 −3 src/core/effects/qgsblureffect.h
  79. +5 −5 src/core/effects/qgscoloreffect.cpp
  80. +3 −3 src/core/effects/qgscoloreffect.h
  81. +4 −4 src/core/effects/qgseffectstack.cpp
  82. +3 −3 src/core/effects/qgseffectstack.h
  83. +10 −10 src/core/effects/qgsgloweffect.cpp
  84. +4 −4 src/core/effects/qgsgloweffect.h
  85. +7 −34 src/core/effects/qgspainteffect.cpp
  86. +5 −5 src/core/effects/qgspainteffect.h
  87. +1 −1 src/core/effects/qgspainteffectregistry.cpp
  88. +4 −4 src/core/effects/qgspainteffectregistry.h
  89. +10 −10 src/core/effects/qgsshadoweffect.cpp
  90. +4 −4 src/core/effects/qgsshadoweffect.h
  91. +6 −6 src/core/effects/qgstransformeffect.cpp
  92. +3 −3 src/core/effects/qgstransformeffect.h
  93. +1 −1 src/core/labeling/qgslabelingengine.cpp
  94. +2 −2 src/core/labeling/qgsrulebasedlabeling.cpp
  95. +1 −1 src/core/labeling/qgsrulebasedlabeling.h
  96. +3 −3 src/core/labeling/qgsvectorlayerlabeling.cpp
  97. +3 −3 src/core/labeling/qgsvectorlayerlabeling.h
  98. +2 −2 src/core/layout/qgscompositionconverter.cpp
  99. +3 −3 src/core/layout/qgslayoutitemmapgrid.cpp
  100. +1 −1 src/core/layout/qgslayoutitemmapoverview.cpp
  101. +1 −1 src/core/layout/qgslayoutitemmarker.cpp
  102. +2 −2 src/core/layout/qgslayoutitemnodeitem.cpp
  103. +1 −1 src/core/layout/qgslayoutitempage.cpp
  104. +1 −1 src/core/layout/qgslayoutitempolygon.cpp
  105. +1 −1 src/core/layout/qgslayoutitempolyline.cpp
  106. +1 −1 src/core/layout/qgslayoutitemshape.cpp
  107. +1 −1 src/core/layout/qgslayoutpagecollection.cpp
  108. +1 −1 src/core/mesh/qgsmeshlayer.cpp
  109. +2 −2 src/core/pointcloud/qgseptdecoder.cpp
  110. +47 −17 src/core/pointcloud/qgspointclouddataprovider.cpp
  111. +4 −0 src/core/pointcloud/qgspointcloudrenderer.h
  112. +20 −0 src/core/processing/qgsprocessingparameters.cpp
  113. +2 −1 src/core/processing/qgsprocessingparameters.h
  114. +2 −2 src/core/providers/ogr/qgsogrprovider.cpp
  115. +2 −2 src/core/qgsabstractdatabaseproviderconnection.cpp
  116. +1 −1 src/core/qgsabstractdatabaseproviderconnection.h
  117. +39 −33 src/core/qgsapplication.cpp
  118. +29 −29 src/core/qgscolorramp.cpp
  119. +13 −12 src/core/qgscolorramp.h
  120. +11 −2 src/core/qgscoordinateutils.cpp
  121. +1 −1 src/core/qgscoordinateutils.h
  122. +1 −1 src/core/qgsdiagramrenderer.cpp
  123. +0 −13 src/core/qgsinterval.cpp
  124. +17 −1 src/core/qgsinterval.h
  125. +1 −1 src/core/qgsmaplayer.cpp
  126. +11 −3 src/core/qgsrenderchecker.cpp
  127. +54 −14 src/core/qgsrenderchecker.h
  128. +159 −0 src/core/qgsrunprocess.cpp
  129. +135 −0 src/core/qgsrunprocess.h
  130. +6 −2 src/core/qgstemporalnavigationobject.cpp
  131. +1 −1 src/core/qgstemporalnavigationobject.h
  132. +52 −0 src/core/qgstemporalutils.cpp
  133. +23 −0 src/core/qgstemporalutils.h
  134. +1 −1 src/core/raster/qgshillshaderenderer.cpp
  135. +1 −1 src/core/raster/qgshillshaderenderer.h
  136. +1 −1 src/core/raster/qgsmultibandcolorrenderer.cpp
  137. +1 −1 src/core/raster/qgsmultibandcolorrenderer.h
  138. +1 −1 src/core/raster/qgspalettedrasterrenderer.cpp
  139. +1 −1 src/core/raster/qgspalettedrasterrenderer.h
  140. +2 −2 src/core/raster/qgsrasterlayer.cpp
  141. +1 −1 src/core/raster/qgsrasterlayer.h
  142. +1 −1 src/core/raster/qgsrasterrenderer.cpp
  143. +1 −1 src/core/raster/qgsrasterrenderer.h
  144. +1 −1 src/core/raster/qgssinglebandgrayrenderer.cpp
  145. +1 −1 src/core/raster/qgssinglebandgrayrenderer.h
  146. +1 −1 src/core/raster/qgssinglebandpseudocolorrenderer.cpp
  147. +1 −1 src/core/raster/qgssinglebandpseudocolorrenderer.h
  148. +2 −2 src/core/symbology/qgs25drenderer.cpp
  149. +14 −14 src/core/symbology/qgsarrowsymbollayer.cpp
  150. +2 −2 src/core/symbology/qgsarrowsymbollayer.h
  151. +5 −5 src/core/symbology/qgscategorizedsymbolrenderer.cpp
  152. +2 −2 src/core/symbology/qgscategorizedsymbolrenderer.h
  153. +29 −29 src/core/symbology/qgsellipsesymbollayer.cpp
  154. +5 −4 src/core/symbology/qgsellipsesymbollayer.h
  155. +135 −135 src/core/symbology/qgsfillsymbollayer.cpp
  156. +25 −25 src/core/symbology/qgsfillsymbollayer.h
  157. +7 −7 src/core/symbology/qgsgeometrygeneratorsymbollayer.cpp
  158. +3 −2 src/core/symbology/qgsgeometrygeneratorsymbollayer.h
  159. +2 −2 src/core/symbology/qgsgraduatedsymbolrenderer.cpp
  160. +1 −1 src/core/symbology/qgsgraduatedsymbolrenderer.h
  161. +48 −48 src/core/symbology/qgslinesymbollayer.cpp
  162. +9 −9 src/core/symbology/qgslinesymbollayer.h
  163. +95 −91 src/core/symbology/qgsmarkersymbollayer.cpp
  164. +19 −17 src/core/symbology/qgsmarkersymbollayer.h
  165. +5 −5 src/core/symbology/qgsmasksymbollayer.cpp
  166. +2 −2 src/core/symbology/qgsmasksymbollayer.h
  167. +1 −1 src/core/symbology/qgspointdistancerenderer.cpp
  168. +1 −1 src/core/symbology/qgspointdistancerenderer.h
  169. +1 −1 src/core/symbology/qgsrenderer.cpp
  170. +2 −2 src/core/symbology/qgsrenderer.h
  171. +3 −3 src/core/symbology/qgsrendererrange.cpp
  172. +1 −1 src/core/symbology/qgsrendererrange.h
  173. +9 −7 src/core/symbology/qgsrulebasedrenderer.cpp
  174. +3 −2 src/core/symbology/qgsrulebasedrenderer.h
  175. +2 −2 src/core/symbology/qgssinglesymbolrenderer.cpp
  176. +1 −1 src/core/symbology/qgssinglesymbolrenderer.h
  177. +55 −29 src/core/symbology/qgssvgcache.h
  178. +4 −4 src/core/symbology/qgssymbol.cpp
  179. +4 −4 src/core/symbology/qgssymbol.h
  180. +10 −10 src/core/symbology/qgssymbollayer.cpp
  181. +6 −5 src/core/symbology/qgssymbollayer.h
  182. +2 −2 src/core/symbology/qgssymbollayerregistry.cpp
  183. +8 −8 src/core/symbology/qgssymbollayerregistry.h
  184. +51 −32 src/core/symbology/qgssymbollayerutils.cpp
  185. +9 −7 src/core/symbology/qgssymbollayerutils.h
  186. +13 −13 src/core/symbology/qgsvectorfieldsymbollayer.cpp
  187. +4 −3 src/core/symbology/qgsvectorfieldsymbollayer.h
  188. +2 −2 src/core/textrenderer/qgstextrenderer.cpp
  189. +2 −2 src/core/vector/qgsvectorlayer.cpp
  190. +1 −1 src/core/vector/qgsvectorlayer.h
  191. +11 −2 src/core/vector/qgsvectorlayereditutils.cpp
  192. +14 −10 src/gui/layertree/qgslayertreeviewdefaultactions.cpp
  193. +4 −4 src/gui/qgslegendpatchshapebutton.cpp
  194. +73 −0 src/gui/qgsmapcanvas.cpp
  195. +15 −0 src/gui/qgsmapcanvas.h
  196. +5 −3 src/gui/qgsmaptoolidentify.cpp
  197. +3 −3 src/gui/qgspropertyassistantwidget.cpp
  198. +4 −4 src/gui/qgssymbolbutton.cpp
  199. +3 −2 src/gui/qgstemporalcontrollerwidget.cpp
  200. +1 −1 src/gui/symbology/qgsdatadefinedsizelegendwidget.cpp
  201. +1 −0 src/providers/mssql/qgsmssqldataitems.cpp
  202. +1 −1 src/providers/mssql/qgsmssqlproviderconnection.cpp
  203. +2 −0 src/providers/pdal/CMakeLists.txt
  204. +6 −1 src/providers/pdal/qgspdaleptgenerationtask.cpp
  205. +14 −13 src/providers/postgres/qgspostgresproviderconnection.cpp
  206. +1 −1 src/server/qgis_mapserver.cpp
  207. +2 −2 src/server/qgscapabilitiescache.cpp
  208. +3 −2 src/server/qgsserverogcapi.cpp
  209. +8 −5 src/server/qgsserverogcapihandler.cpp
  210. +12 −6 src/server/qgsserverparameters.cpp
  211. +3 −1 src/server/qgsserverplugins.cpp
  212. +10 −1 src/server/qgsserverprojectutils.cpp
  213. +1 −1 src/server/qgsserverprojectutils.h
  214. +2 −3 src/server/qgsserverquerystringparameter.cpp
  215. +3 −2 src/server/qgsserversettings.cpp
  216. +2 −1 src/server/qgsservicenativeloader.cpp
  217. +4 −3 src/server/services/landingpage/qgslandingpageutils.cpp
  218. +12 −29 src/server/services/wms/qgswmsgetcapabilities.cpp
  219. +1 −1 src/server/services/wms/qgswmsgetstyles.cpp
  220. +1 −1 src/ui/qgisapp.ui
  221. +13 −0 tests/src/analysis/testqgsprocessing.cpp
  222. +14 −1 tests/src/app/testqgsapplocatorfilters.cpp
  223. +2 −2 tests/src/core/testqgsdatadefinedsizelegend.cpp
  224. +5 −5 tests/src/core/testqgsdiagram.cpp
  225. +1 −1 tests/src/core/testqgsdxfexport.cpp
  226. +1 −1 tests/src/core/testqgslabelingengine.cpp
  227. +4 −4 tests/src/core/testqgslayertree.cpp
  228. +2 −2 tests/src/core/testqgslayoutatlas.cpp
  229. +1 −1 tests/src/core/testqgslegendrenderer.cpp
  230. +2 −2 tests/src/core/testqgslinefillsymbol.cpp
  231. +2 −2 tests/src/core/testqgsmarkerlinesymbol.cpp
  232. +11 −11 tests/src/core/testqgspainteffect.cpp
  233. +4 −4 tests/src/core/testqgspainteffectregistry.cpp
  234. +7 −7 tests/src/core/testqgspointpatternfillsymbol.cpp
  235. +14 −14 tests/src/core/testqgsstyle.cpp
  236. +7 −1 tests/src/core/testqgstemporalnavigationobject.cpp
  237. +1 −0 tests/src/python/CMakeLists.txt
  238. +86 −0 tests/src/python/test_qgsblockingprocess.py
  239. +1 −1 tests/src/python/test_qgsserver.py
  240. +59 −1 tests/src/python/test_qgstemporalutils.py
  241. BIN tests/testdata/curved_polys.gpkg
  242. +2 −2 tests/testdata/qgis_server/getcapabilities.txt
  243. +2 −2 tests/testdata/qgis_server/getcapabilities_inspire.txt
  244. +2 −2 tests/testdata/qgis_server/getprojectsettings.txt
  245. +2 −2 tests/testdata/qgis_server/getprojectsettings_opacity.txt
  246. +1 −1 tests/testdata/qgis_server/landingpage/test_landing_page_index.json
  247. +1 −1 tests/testdata/qgis_server/landingpage/test_landing_page_with_pg_index.json
  248. +1 −1 tests/testdata/qgis_server/landingpage/test_project_QGIS_Server_-_Grouped_Nested_Layer.json
  249. +2 −2 tests/testdata/qgis_server/wms_getcapabilities_1_1_1.txt
  250. +2 −2 tests/testdata/qgis_server/wms_getcapabilities_1_3_0.txt
  251. +2 −1 tests/testdata/qgis_server/wms_getcapabilities_empty_layer.txt
  252. +2 −1 tests/testdata/qgis_server/wms_getcapabilities_empty_spatial_layer.txt
  253. +2 −1 tests/testdata/qgis_server/wms_getcapabilities_without_title.txt
  254. +2 −2 tests/testdata/qgis_server_accesscontrol/results/getcapabilities_wms_dimension.txt
@@ -56,8 +56,9 @@ Build-Depends:
python3-pyqt5.qsci,
python3-pyqt5.qtsql,
python3-pyqt5.qtsvg,
python3-sip-dev,
#sid# sip5-tools,
#buster bullseye bionic focal groovy# python3-sip-dev, python3-sip,
#buster bullseye bionic focal groovy# python3-sip,
python3-termcolor,
python3-yaml,
qt3d5-dev,
@@ -72,6 +72,7 @@ Milena Nowotarska
Minoru Akagi
Nikos Alexandris
Paolo Cavallini
Patrice Pineault
Paul Blottiere
Paul Ramsey
Peter Petrik
@@ -29,6 +29,7 @@ void BuPyramid::run(const Options& options, ProgressWriter& progress)
{
m_b.inputDir = options.tempDir;
m_b.outputDir = options.outputDir;
m_b.stats = options.stats;

readBaseInfo();
getInputFiles();
@@ -164,7 +165,31 @@ void BuPyramid::writeInfo()
out << "\"type\": \"" << typeString(pdal::Dimension::base(fdi.type)) << "\", ";
if (fdi.name == "X" || fdi.name == "Y" || fdi.name == "Z")
out << "\"scale\": 0.01, \"offset\": 0, ";
out << "\"size\": " << pdal::Dimension::size(fdi.type) << " ";
out << "\"size\": " << pdal::Dimension::size(fdi.type);
const Stats *stats = m_manager.stats(fdi.name);
if (stats)
{
const Stats::EnumMap& v = stats->values();
out << ", ";
if (v.size())
{
out << "\"counts\": [ ";
for (auto ci = v.begin(); ci != v.end(); ++ci)
{
auto c = *ci;
if (ci != v.begin())
out << ", ";
out << "{\"value\": " << c.first << ", \"count\": " << c.second << "}";
}
out << "], ";
}
out << "\"count\": " << m_manager.totalPoints() << ", ";
out << "\"maximum\": " << stats->maximum() << ", ";
out << "\"minimum\": " << stats->minimum() << ", ";
out << "\"mean\": " << stats->average() << ", ";
out << "\"stddev\": " << stats->stddev() << ", ";
out << "\"variance\": " << stats->variance();
}
out << "}";
if (di + 1 != m_b.dimInfo.end())
out << ",";
@@ -35,6 +35,7 @@ struct BaseInfo
int maxLevel;
DimInfoList dimInfo;
pdal::SpatialReference srs;
bool stats;
};

} // namespace bu
@@ -124,7 +124,7 @@ std::cerr << m_vi.key() << " Accepted/Rejected/num points = " <<
**/

// If this is the final key, append any remaining file infos as accepted points and
// write the accepted points as binary.
// write the accepted points as compressed.
if (m_vi.key() == VoxelKey(0, 0, 0, 0))
{
appendRemainder(accepted);
@@ -136,8 +136,11 @@ std::cerr << m_vi.key() << " Accepted/Rejected/num points = " <<
}


bool Processor::acceptable(int /* pointId */, GridKey key)
bool Processor::acceptable(int pointId, GridKey key)
{
//ABELL - Currently unused - see commented-out code.
(void)pointId;

VoxelInfo::Grid& grid = m_vi.grid();

auto it = grid.find(key);
@@ -258,6 +261,9 @@ void Processor::writeCompressedOutput(Index& index)
std::sort(index.begin(), index.end());

IndexIter pos = index.begin();

// If any of our octants has points, we have to write the parent octant, whether or not
// it contains points, in order to create a full tree.
for (int octant = 0; octant < 8; ++octant)
if (m_vi[octant].hasPoints() || m_vi[octant].mustWrite())
{
@@ -267,26 +273,47 @@ void Processor::writeCompressedOutput(Index& index)
}


// o Octant we're writing.
// index Index of all rejected points that were rejected and not hoisted into the parent.
// pos Start position of this octant's point in the index.
// \return Position of the first point in the next octant of our index.
Processor::IndexIter
Processor::writeOctantCompressed(const OctantInfo& o, Index& index, IndexIter pos)
{
auto begin = pos;
pdal::PointTable table;
IndexedStats stats;

//ABELL - fixme
// For now we copy the dimension list so we're sure that it matches the layout, though
// there's no reason why it should change. We should modify things to use a single
// layout.
DimInfoList dims = m_b.dimInfo;
for (FileDimInfo& fdi : dims)
{
fdi.dim = table.layout()->registerOrAssignDim(fdi.name, fdi.type);
if (m_b.stats)
{
if (fdi.dim == pdal::Dimension::Id::Classification)
stats.push_back({fdi.dim, Stats(fdi.name, Stats::EnumType::Enumerate, false)});
else
stats.push_back({fdi.dim, Stats(fdi.name, Stats::EnumType::NoEnum, false)});
}
}
table.finalize();

pdal::PointViewPtr view(new pdal::PointView(table));

// The octant's points can came from one or more FileInfo. The points are sorted such
// all the points that come from a single FileInfo are consecutive.
auto fii = o.fileInfos().begin();
auto fiiEnd = o.fileInfos().end();
size_t count = 0;
if (fii != fiiEnd)
{
// We're trying to find the range of points that come from a single FileInfo.
// If pos is the end of the index of the the current file info, append the points
// to the view. Otherwise, advance the position.
while (true)
{
if (pos == index.end() || *pos >= fii->start() + fii->numPoints())
@@ -296,6 +323,9 @@ Processor::writeOctantCompressed(const OctantInfo& o, Index& index, IndexIter po
if (pos == index.end())
break;
begin = pos;

// Advance through file infos as long as we don't have points that
// correspond to it.
do
{
fii++;
@@ -309,14 +339,14 @@ Processor::writeOctantCompressed(const OctantInfo& o, Index& index, IndexIter po
flush:
try
{
flushCompressed(table, view, o);
flushCompressed(table, view, o, stats);
}
catch (pdal::pdal_error& err)
{
fatal(err.what());
}

m_manager.logOctant(o.key(), count);
m_manager.logOctant(o.key(), count, stats);
return pos;
}

@@ -340,25 +370,39 @@ void Processor::appendCompressed(pdal::PointViewPtr view, const DimInfoList& dim


void Processor::flushCompressed(pdal::PointTableRef table, pdal::PointViewPtr view,
const OctantInfo& oi)
const OctantInfo& oi, IndexedStats& stats)
{
using namespace pdal;

std::string filename = m_b.outputDir + "/ept-data/" + oi.key().toString() + ".laz";

if (m_b.stats)
{
for (PointId id = 0; id < view->size(); ++id)
{
for (auto& sp : stats)
{
Dimension::Id dim = sp.first;
Stats& s = sp.second;
s.insert(view->getFieldAs<double>(dim, id));
}
}
}

StageFactory factory;

BufferReader r;
r.addView(view);

Stage *prev = &r;

if (table.layout()->hasDim(Dimension::Id::GpsTime))
{
Stage *f = factory.createStage("filters.sort");
pdal::Options fopts;
fopts.add("dimension", "gpstime");
f->setOptions(fopts);
f->setInput(r);
f->setInput(*prev);
prev = f;
}

@@ -17,6 +17,7 @@

#include "BuTypes.hpp"
#include "PointAccessor.hpp"
#include "Stats.hpp"
#include "VoxelInfo.hpp"

namespace untwine
@@ -53,7 +54,7 @@ class Processor
void appendCompressed(pdal::PointViewPtr view, const DimInfoList& dims, const FileInfo& fi,
IndexIter begin, IndexIter end);
void flushCompressed(pdal::PointTableRef table, pdal::PointViewPtr view,
const OctantInfo& oi);
const OctantInfo& oi, IndexedStats& stats);

VoxelInfo m_vi;
const BaseInfo& m_b;
@@ -28,8 +28,6 @@ namespace untwine
namespace bu
{

//PyramidManager::PyramidManager(const BaseInfo& b) : m_b(b), m_pool(6), m_totalPoints(0)
//PyramidManager::PyramidManager(const BaseInfo& b) : m_b(b), m_pool(8), m_totalPoints(0)
PyramidManager::PyramidManager(const BaseInfo& b) : m_b(b), m_pool(10), m_totalPoints(0)
{}

@@ -136,10 +134,23 @@ OctantInfo PyramidManager::removeComplete(const VoxelKey& k)
}


void PyramidManager::logOctant(const VoxelKey& k, int cnt)
void PyramidManager::logOctant(const VoxelKey& k, int cnt, const IndexedStats& istats)
{
std::lock_guard<std::mutex> lock(m_mutex);

for (auto is : istats)
{
Stats& s = is.second;

auto it = m_stats.find(s.name());
if (it != m_stats.end())
{
Stats& cur = it->second;
cur.merge(s);
}
else
m_stats.insert({s.name(), s});
}
m_written.insert({k, cnt});
m_totalPoints += cnt;
}
@@ -229,5 +240,14 @@ std::deque<VoxelKey> PyramidManager::emit(const VoxelKey& p, int stopLevel, Entr
return roots;
}


Stats *PyramidManager::stats(const std::string& name)
{
auto si = m_stats.find(name);
if (si == m_stats.end())
return nullptr;
return &si->second;
}

} // namespace bu
} // namespace untwine
@@ -19,6 +19,7 @@

#include "BuTypes.hpp"
#include "OctantInfo.hpp"
#include "Stats.hpp"
#include "../untwine/ThreadPool.hpp"

namespace untwine
@@ -42,9 +43,10 @@ class PyramidManager
void setProgress(ProgressWriter *progress);
void queue(const OctantInfo& o);
void run();
void logOctant(const VoxelKey& k, int cnt);
void logOctant(const VoxelKey& k, int cnt, const IndexedStats& istats);
uint64_t totalPoints() const
{ return m_totalPoints; }
Stats *stats(const std::string& name);

private:
const int LevelBreak = 4;
@@ -56,6 +58,7 @@ class PyramidManager
std::queue<OctantInfo> m_queue;
ThreadPool m_pool;
uint64_t m_totalPoints;
std::map<std::string, Stats> m_stats;
ProgressWriter *m_progress;
//
std::unordered_map<VoxelKey, int> m_written;
@@ -0,0 +1,67 @@

#include "Stats.hpp"

#include <cmath>

namespace untwine
{

void Stats::computeGlobalStats()
{
auto compute_median = [](std::vector<double> vals)
{
std::nth_element(vals.begin(), vals.begin() + vals.size() / 2, vals.end());
return *(vals.begin() + vals.size() / 2);
};

// TODO add quantiles
m_median = compute_median(m_data);
std::transform(m_data.begin(), m_data.end(), m_data.begin(),
[this](double v) { return std::fabs(v - this->m_median); });
m_mad = compute_median(m_data);
}

// Math comes from https://prod.sandia.gov/techlib-noauth/access-control.cgi/2008/086212.pdf
// (Pebay paper from Sandia labs, 2008)
bool Stats::merge(const Stats& s)
{
if ((m_name != s.m_name) || (m_enumerate != s.m_enumerate) || (m_advanced != s.m_advanced))
return false;

double n1 = m_cnt;
double n2 = s.m_cnt;
double n = n1 + n2;
double nsq = n * n;
double n1n2 = m_cnt * s.m_cnt;
double n1sq = n1 * n1;
double n2sq = n2 * n2;
double ncube = n * n * n;
double deltaMean = s.M1 - M1;

if (n == 0)
return true;

double m1 = M1 + s.m_cnt * deltaMean / n;
double m2 = M2 + s.M2 + n1n2 * std::pow(deltaMean, 2) / n;
double m3 = M3 + s.M3 + n1n2 * (n1 - n2) * std::pow(deltaMean, 3) / nsq +
3 * (n1 * s.M2 - n2 * M2) * deltaMean / n;
double m4 = M4 + s.M4 +
n1n2 * (n1sq - n1n2 + n2sq) * std::pow(deltaMean, 4) / ncube +
6 * (n1sq * s.M2 + n2sq * M2) * std::pow(deltaMean, 2) / nsq +
4 * (n1 * s.M3 - n2 * M3) * deltaMean / n;

M1 = m1;
M2 = m2;
M3 = m3;
M4 = m4;
m_min = (std::min)(m_min, s.m_min);
m_max = (std::max)(m_max, s.m_max);
m_cnt = s.m_cnt + m_cnt;
m_data.insert(m_data.begin(), s.m_data.begin(), s.m_data.end());
for (auto p : s.m_values)
m_values[p.first] += p.second;

return true;
}

} // namespace untwine

0 comments on commit cf5180a

Please sign in to comment.
You can’t perform that action at this time.