Skip to content
Permalink
Browse files

Memory dataset group and possibility to persist it (#37389)

[FEATURE] Introduces memory dataset groups for mesh layer. These dataset groups are temporary and are not kept when the project is closed.

Memory dataset groups can be created from the mesh calculator with a new option.

Allows the possibility to remove or save these memory dataset groups to a file with specified driver.
  • Loading branch information
vcloarec committed Jul 1, 2020
1 parent 6068c64 commit 6be16a5bfe81fe8dddb30b4c0fb75ad5772c6558
Showing with 5,154 additions and 1,292 deletions.
  1. +20 −2 external/mdal/api/mdal.h
  2. +66 −29 external/mdal/frmts/mdal_2dm.cpp
  3. +10 −11 external/mdal/frmts/mdal_2dm.hpp
  4. +2 −0 external/mdal/frmts/mdal_3di.cpp
  5. +1 −0 external/mdal/frmts/mdal_3di.hpp
  6. +8 −1 external/mdal/frmts/mdal_ascii_dat.cpp
  7. +2 −0 external/mdal/frmts/mdal_ascii_dat.hpp
  8. +5 −0 external/mdal/frmts/mdal_binary_dat.cpp
  9. +2 −0 external/mdal/frmts/mdal_binary_dat.hpp
  10. +24 −20 external/mdal/frmts/mdal_cf.cpp
  11. +4 −2 external/mdal/frmts/mdal_cf.hpp
  12. +6 −0 external/mdal/frmts/mdal_driver.cpp
  13. +2 −0 external/mdal/frmts/mdal_driver.hpp
  14. +3 −7 external/mdal/frmts/mdal_esri_tin.cpp
  15. +2 −6 external/mdal/frmts/mdal_flo2d.cpp
  16. +2 −6 external/mdal/frmts/mdal_gdal.cpp
  17. +2 −6 external/mdal/frmts/mdal_hec2d.cpp
  18. +1,054 −397 external/mdal/frmts/mdal_selafin.cpp
  19. +278 −46 external/mdal/frmts/mdal_selafin.hpp
  20. +3 −7 external/mdal/frmts/mdal_sww.cpp
  21. +3 −1 external/mdal/frmts/mdal_tuflowfv.cpp
  22. +1 −0 external/mdal/frmts/mdal_tuflowfv.hpp
  23. +33 −6 external/mdal/frmts/mdal_ugrid.cpp
  24. +3 −1 external/mdal/frmts/mdal_ugrid.hpp
  25. +3 −7 external/mdal/frmts/mdal_xms_tin.cpp
  26. +25 −2 external/mdal/mdal.cpp
  27. +10 −28 external/mdal/mdal_data_model.cpp
  28. +8 −14 external/mdal/mdal_data_model.hpp
  29. +18 −1 external/mdal/mdal_datetime.cpp
  30. +4 −1 external/mdal/mdal_datetime.hpp
  31. +30 −14 external/mdal/mdal_memory_data_model.cpp
  32. +25 −7 external/mdal/mdal_memory_data_model.hpp
  33. +12 −0 external/mdal/mdal_utils.cpp
  34. +16 −1 external/mdal/mdal_utils.hpp
  35. +39 −1 python/analysis/auto_generated/mesh/qgsmeshcalculator.sip.in
  36. +40 −0 python/core/auto_generated/mesh/qgsmeshdataprovider.sip.in
  37. +57 −0 python/core/auto_generated/mesh/qgsmeshdataset.sip.in
  38. +208 −1 python/core/auto_generated/mesh/qgsmeshlayer.sip.in
  39. +3 −2 python/core/auto_generated/qgsmaplayer.sip.in
  40. +7 −1 python/core/auto_generated/qgsprovidermetadata.sip.in
  41. +4 −4 src/3d/mesh/qgsmesh3dgeometry_p.cpp
  42. +2 −2 src/3d/mesh/qgsmesh3dmaterial_p.cpp
  43. +0 −2 src/analysis/mesh/qgsmeshcalcnode.h
  44. +62 −16 src/analysis/mesh/qgsmeshcalculator.cpp
  45. +38 −3 src/analysis/mesh/qgsmeshcalculator.h
  46. +38 −39 src/analysis/mesh/qgsmeshcalcutils.cpp
  47. +0 −3 src/analysis/mesh/qgsmeshcalcutils.h
  48. +99 −39 src/app/mesh/qgsmeshcalculatordialog.cpp
  49. +9 −5 src/app/mesh/qgsmeshcalculatordialog.h
  50. +223 −6 src/app/mesh/qgsmeshdatasetgrouptreeview.cpp
  51. +57 −2 src/app/mesh/qgsmeshdatasetgrouptreeview.h
  52. +2 −1 src/app/mesh/qgsmeshdatasetgrouptreewidget.cpp
  53. +1 −0 src/app/mesh/qgsmeshdatasetgrouptreewidget.h
  54. +1 −1 src/app/mesh/qgsmeshrendereractivedatasetwidget.cpp
  55. +6 −6 src/app/mesh/qgsmeshrendererscalarsettingswidget.cpp
  56. +3 −5 src/app/mesh/qgsmeshrenderervectorsettingswidget.cpp
  57. +13 −13 src/app/mesh/qgsmeshstaticdatasetwidget.cpp
  58. +1 −2 src/app/mesh/qgsrenderermeshpropertieswidget.cpp
  59. +25 −21 src/app/qgisapp.cpp
  60. +2 −2 src/core/CMakeLists.txt
  61. +30 −1 src/core/mesh/qgsmeshdataprovider.cpp
  62. +40 −3 src/core/mesh/qgsmeshdataprovider.h
  63. +2 −2 src/core/mesh/qgsmeshdataprovidertemporalcapabilities.cpp
  64. +1 −0 src/core/mesh/qgsmeshdataprovidertemporalcapabilities.h
  65. +274 −5 src/core/mesh/qgsmeshdataset.cpp
  66. +246 −0 src/core/mesh/qgsmeshdataset.h
  67. +684 −0 src/core/mesh/qgsmeshdatasetgroupstore.cpp
  68. +227 −0 src/core/mesh/qgsmeshdatasetgroupstore.h
  69. +145 −80 src/core/mesh/qgsmeshlayer.cpp
  70. +186 −6 src/core/mesh/qgsmeshlayer.h
  71. +7 −7 src/core/mesh/qgsmeshlayerrenderer.cpp
  72. +8 −8 src/core/mesh/qgsmeshlayerutils.cpp
  73. +0 −43 src/core/mesh/qgsmeshlayerutils.h
  74. +47 −161 src/core/providers/meshmemory/qgsmeshmemorydataprovider.cpp
  75. +6 −36 src/core/providers/meshmemory/qgsmeshmemorydataprovider.h
  76. +3 −2 src/core/qgsmaplayer.h
  77. +7 −2 src/core/qgsprovidermetadata.cpp
  78. +8 −1 src/core/qgsprovidermetadata.h
  79. +3 −3 src/gui/qgsmaptoolidentify.cpp
  80. +106 −9 src/providers/mdal/qgsmdalprovider.cpp
  81. +6 −0 src/providers/mdal/qgsmdalprovider.h
  82. +41 −27 src/ui/mesh/qgsmeshcalculatordialogbase.ui
  83. +2 −2 tests/src/analysis/testqgsmeshcalculator.cpp
  84. +448 −94 tests/src/core/testqgsmeshlayer.cpp
@@ -163,6 +163,14 @@ MDAL_EXPORT bool MDAL_DR_meshLoadCapability( MDAL_DriverH driver );
*/
MDAL_EXPORT bool MDAL_DR_writeDatasetsCapability( MDAL_DriverH driver, MDAL_DataLocation location );

/**
* Returns the file suffix used to write datasets on file
* not thread-safe and valid only till next call
*
* \since MDAL 0.7.0
*/
MDAL_EXPORT const char *MDAL_DR_writeDatasetsSuffix( MDAL_DriverH driver );

/**
* Returns whether driver has capability to save mesh
*/
@@ -208,8 +216,10 @@ MDAL_EXPORT MDAL_MeshH MDAL_LoadMesh( const char *uri );
* not thread-safe and valid only till next call
*
* Parameter uri can be in format:
* - <drivername>:"meshfile" - function then returns uris with provided driver and meshfile
* - "meshfile" or meshfile - function then finds proper driver and returns uris with it
*
* - <drivername>:"meshfile" - function then returns uris with provided driver and meshfile
* - "meshfile" or meshfile - function then finds proper driver and returns uris with it
*
* The uris can be used directly in MDAL_LoadMesh to load particular meshes
*
* \since MDAL 0.6.0
@@ -530,6 +540,14 @@ MDAL_EXPORT const char *MDAL_G_referenceTime( MDAL_DatasetGroupH group );
*/
MDAL_EXPORT bool MDAL_G_isTemporal( MDAL_DatasetGroupH group );

/**
* Returns dataset group uri
* not thread-safe and valid only till next call
*
* \since MDAL 0.7.0
*/
MDAL_EXPORT const char *MDAL_G_uri( MDAL_DatasetGroupH group );

///////////////////////////////////////////////////////////////////////////////////////
/// DATASETS
///////////////////////////////////////////////////////////////////////////////////////
@@ -22,19 +22,11 @@

#define DRIVER_NAME "2DM"

MDAL::Mesh2dm::Mesh2dm( size_t verticesCount,
size_t edgesCount,
size_t facesCount,
size_t faceVerticesMaximumCount,
MDAL::BBox extent,
MDAL::Mesh2dm::Mesh2dm( size_t faceVerticesMaximumCount,
const std::string &uri,
const std::map<size_t, size_t> vertexIDtoIndex )
: MemoryMesh( DRIVER_NAME,
verticesCount,
edgesCount,
facesCount,
faceVerticesMaximumCount,
extent,
uri )
, mVertexIDtoIndex( vertexIDtoIndex )
{
@@ -125,6 +117,8 @@ std::unique_ptr<MDAL::Mesh> MDAL::Driver2dm::load( const std::string &meshFile,
size_t faceCount = 0;
size_t vertexCount = 0;
size_t edgesCount = 0;
size_t materialCount = 0;
bool hasMaterialsDefinitionsForElements = false;

// Find out how many nodes and elements are contained in the .2dm mesh file
while ( std::getline( in, line ) )
@@ -150,15 +144,21 @@ std::unique_ptr<MDAL::Mesh> MDAL::Driver2dm::load( const std::string &meshFile,
MDAL::Log::warning( MDAL_Status::Err_UnsupportedElement, name(), "found unsupported element" );
return nullptr;
}
// If specified, update the number of materials of the mesh
else if ( startsWith( line, "NUM_MATERIALS_PER_ELEM" ) )
{
hasMaterialsDefinitionsForElements = true;
materialCount = MDAL::toSizeT( split( line, ' ' )[1] );
}
}

// Allocate memory
Vertices vertices( vertexCount );
Edges edges( edgesCount );
Faces faces( faceCount );

// Basement 3.x supports definition of elevation for cell centers
std::vector<double> elementCenteredElevation;
// .2dm mesh files may have any number of material ID columns
std::vector<std::vector<double>> faceMaterials;

in.clear();
in.seekg( 0, std::ios::beg );
@@ -187,26 +187,46 @@ std::unique_ptr<MDAL::Mesh> MDAL::Driver2dm::load( const std::string &meshFile,
face.resize( faceVertexCount );

// chunks format here
// E** id vertex_id1, vertex_id2, ... material_id (elevation - optional)
// E** id vertex_id1, vertex_id2, vertex_id3, material_id [, aux_column_1, aux_column_2, ...]
// vertex ids are numbered from 1
// Right now we just store node IDs here - we will convert them to node indices afterwards
assert( chunks.size() > faceVertexCount + 1 );

for ( size_t i = 0; i < faceVertexCount; ++i )
face[i] = MDAL::toSizeT( chunks[i + 2] ) - 1; // 2dm is numbered from 1

// OK, now find out if there is optional cell elevation (BASEMENT 3.x)
if ( chunks.size() == faceVertexCount + 4 )
// NUM_MATERIALS_PER_ELEM tag provided, use new MATID parser
if ( hasMaterialsDefinitionsForElements )
{
// This assertion will fail if a mesh has fewer material ID columns than
// promised by the NUM_MATERIAL_PER_ELEM tag.
assert( chunks.size() - 5 >= materialCount );

if ( faceMaterials.empty() ) // Initialize dataset if still empty
{
faceMaterials = std::vector<std::vector<double>>( materialCount, std::vector<double>(
faceCount, std::numeric_limits<double>::quiet_NaN() ) );
}

// Add material ID values
for ( size_t i = 0; i < materialCount; ++i )
{
// Offset of 2 for E** tag and element ID
faceMaterials[i][faceIndex] = MDAL::toDouble( chunks[ faceVertexCount + 2 + i] );
}
}

// initialize dataset if it is still empty
if ( elementCenteredElevation.empty() )
// No NUM_MATERIALS_PER_ELEM tag provided, use legacy MATID parser
else if ( chunks.size() == faceVertexCount + 4 )
{
if ( faceMaterials.empty() ) // Initialize dataset if still empty
{
elementCenteredElevation = std::vector<double>( faceCount, std::numeric_limits<double>::quiet_NaN() );
// Add a single vector dataset for the "Bed Elevation (Face)" dataset
faceMaterials = std::vector<std::vector<double>>( 1, std::vector<double>(
faceCount, std::numeric_limits<double>::quiet_NaN() ) );
}

// add Bed Elevation (Face) value
elementCenteredElevation[faceIndex] = MDAL::toDouble( chunks[ faceVertexCount + 3 ] );
faceMaterials[0][faceIndex] = MDAL::toDouble( chunks[ faceVertexCount + 3 ] );
}

faceIndex++;
@@ -276,22 +296,39 @@ std::unique_ptr<MDAL::Mesh> MDAL::Driver2dm::load( const std::string &meshFile,

std::unique_ptr< Mesh2dm > mesh(
new Mesh2dm(
vertices.size(),
edges.size(),
faces.size(),
MAX_VERTICES_PER_FACE_2DM,
computeExtent( vertices ),
mMeshFile,
vertexIDtoIndex
)
);
mesh->faces = faces;
mesh->vertices = vertices;
mesh->edges = edges;
mesh->setFaces( std::move( faces ) );
mesh->setVertices( std::move( vertices ) );
mesh->setEdges( std::move( edges ) );

// Add Bed Elevation
MDAL::addBedElevationDatasetGroup( mesh.get(), mesh->vertices() );

// Add material IDs
if ( hasMaterialsDefinitionsForElements )
{
// New MATID parser: Add all MATID dataset groups
std::string dataSetName;
for ( size_t i = 0; i < materialCount; ++i )
{
// The first two columns get special names for convenience
if ( i == 0 ) dataSetName = "Material ID";
else if ( i == 1 ) dataSetName = "Bed Elevation (Face)";
else dataSetName = "Auxiliary Material ID " + std::to_string( i - 1 );

// Add Bed Elevations
MDAL::addFaceScalarDatasetGroup( mesh.get(), elementCenteredElevation, "Bed Elevation (Face)" );
MDAL::addBedElevationDatasetGroup( mesh.get(), vertices );
MDAL::addFaceScalarDatasetGroup( mesh.get(), faceMaterials[i], dataSetName );
}
}
// Add "Bed Elevation (Face)"
else if ( !faceMaterials.empty() )
{
// Legacy MATID parser: "Bed Elevation (Face)" dataset group only
MDAL::addFaceScalarDatasetGroup( mesh.get(), faceMaterials[0], "Bed Elevation (Face)" );
}

return std::unique_ptr<Mesh>( mesh.release() );
}
@@ -22,11 +22,7 @@ namespace MDAL
class Mesh2dm: public MemoryMesh
{
public:
Mesh2dm( size_t verticesCount,
size_t edgesCount,
size_t facesCount,
size_t faceVerticesMaximumCount,
BBox extent,
Mesh2dm( size_t faceVerticesMaximumCount,
const std::string &uri,
const std::map<size_t, size_t> vertexIDtoIndex
);
@@ -63,12 +59,15 @@ namespace MDAL
*
* full specification here: https://www.xmswiki.com/wiki/SMS:2D_Mesh_Files_*.2dm
*
* Exception for the official specification is for recognition of cell-centered
* elevation values supported by BASEMENT 3.x releases
* If face definition has extra column, it is parsed and recognized as
* elevation, e.g. format for triangle
* E3T id 1 2 3 mat_id elevation
* and added automatically as "Bed Elevation (Face)"
* This will process as many material IDs as promised by the NUM_MATERIALS_PER_ELEM tag and add them as face
* dataset groups. The naming for these groups is "Material ID" for the first, "Bed Elevation (Face)" for the
* second, and finally "Auxiliary Material ID <X>" for any subsequent materials, X being a counter to ensure
* unique group names:
* E** id 1 2 3 [Material ID] [Bed Elevation (Face)] [Auxiliary Material ID 1] [Auxiliary Material ID 2] ...
* If the NUM_MATERIALS_PER_ELEM tag is not provided, a fallback mode is used that will only check for the
* second MATID column and add it under the name "Bed Elevation (Face)" if found.
* Noe that this is purely a compatibility mode for BASEMENT 3.x releases; NUM_MATERIALS_... is a required
* tag according to the 2DM specification.
*
* Note that some 2dm formats do have some extra columns after mat_id column with
* data with unknown origin/name (e.g. tests/data/2dm/regular_grid.2dm)
@@ -268,11 +268,13 @@ void MDAL::Driver3Di::parseNetCDFVariableMetadata( int varid,
std::string &name,
bool *is_vector,
bool *isPolar,
bool *invertedDirection,
bool *is_x )
{
*is_vector = false;
*is_x = true;
*isPolar = false;
MDAL_UNUSED( invertedDirection )

std::string long_name = mNcFile->getAttrStr( "long_name", varid );
if ( long_name.empty() )
@@ -58,6 +58,7 @@ namespace MDAL
std::string &name,
bool *is_vector,
bool *isPolar,
bool *invertedDirection,
bool *is_x ) override;
std::vector<std::pair<double, double>> parseClassification( int varid ) const override;

@@ -485,7 +485,9 @@ bool MDAL::DriverAsciiDat::persist( MDAL::DatasetGroup *group )
if ( !MDAL::contains( uri, "_els" ) && group->dataLocation() != MDAL_DataLocation::DataOnVertices )
{
// Should contain _els in name for edges/faces dataset but it does not
uri.insert( uri.size() - 4, "_els" );
int pos = uri.size() - 4;
uri.insert( std::max( 0, pos ), "_els" );
group->replaceUri( uri );
}

if ( ( mesh->facesCount() > 0 ) && ( mesh->edgesCount() > 0 ) )
@@ -561,3 +563,8 @@ bool MDAL::DriverAsciiDat::persist( MDAL::DatasetGroup *group )

return false;
}

std::string MDAL::DriverAsciiDat::writeDatasetOnFileSuffix() const
{
return "dat";
}
@@ -59,6 +59,8 @@ namespace MDAL
void load( const std::string &datFile, Mesh *mesh ) override;
bool persist( DatasetGroup *group ) override;

std::string writeDatasetOnFileSuffix() const override;

private:
bool canReadOldFormat( const std::string &line ) const;
bool canReadNewFormat( const std::string &line ) const;
@@ -491,3 +491,8 @@ bool MDAL::DriverBinaryDat::persist( MDAL::DatasetGroup *group )

return false;
}

std::string MDAL::DriverBinaryDat::writeDatasetOnFileSuffix() const
{
return "dat";
}
@@ -31,6 +31,8 @@ namespace MDAL
void load( const std::string &datFile, Mesh *mesh ) override;
bool persist( DatasetGroup *group ) override;

std::string writeDatasetOnFileSuffix() const override;

private:
bool readVertexTimestep( const Mesh *mesh,
std::shared_ptr<DatasetGroup> group,

0 comments on commit 6be16a5

Please sign in to comment.
You can’t perform that action at this time.