Skip to content

Commit

Permalink
update MDAL to 0.1.4 (RC1 for QGIS 3.6)
Browse files Browse the repository at this point in the history
  • Loading branch information
@PeterPetrik @nyalldawson
PeterPetrik authored and nyalldawson committed Jan 22, 2019
1 parent 4048d37 commit 928a559
Show file tree
Hide file tree
Showing 18 changed files with 1,088 additions and 586 deletions.
2 changes: 2 additions & 0 deletions external/mdal/frmts/mdal_3di.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
*/

#include "mdal_3di.hpp"
#include <netcdf.h>
#include <assert.h>

MDAL::Driver3Di::Driver3Di()
: DriverCF(
Expand Down
201 changes: 130 additions & 71 deletions external/mdal/frmts/mdal_ascii_dat.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,73 +49,101 @@ bool MDAL::DriverAsciiDat::canRead( const std::string &uri )
}
line = trim( line );

if ( line != "DATASET" &&
line != "SCALAR" &&
line != "VECTOR" )
{
return false;
}
return true;
return canReadNewFormat( line ) || canReadOldFormat( line );
}

bool MDAL::DriverAsciiDat::canReadOldFormat( const std::string &line ) const
{
return MDAL::contains( line, "SCALAR" ) ||
MDAL::contains( line, "VECTOR" ) ||
MDAL::contains( line, "TS" );
}

/**
* The DAT format contains "datasets" and each dataset has N-outputs. One output
* represents data for all vertices/faces for one timestep
*
* In MDAL we convert one output to one MDAL dataset;
*
*/
void MDAL::DriverAsciiDat::load( const std::string &datFile, MDAL::Mesh *mesh, MDAL_Status *status )
bool MDAL::DriverAsciiDat::canReadNewFormat( const std::string &line ) const
{
mDatFile = datFile;
if ( status ) *status = MDAL_Status::None;
return line == "DATASET";
}

if ( !MDAL::fileExists( mDatFile ) )
{
if ( status ) *status = MDAL_Status::Err_FileNotFound;
return;
}

std::ifstream in( mDatFile, std::ifstream::in );
void MDAL::DriverAsciiDat::loadOldFormat( std::ifstream &in,
Mesh *mesh,
MDAL_Status *status ) const
{
std::shared_ptr<DatasetGroup> group; // DAT outputs data
std::string groupName( MDAL::baseName( mDatFile ) );
std::string line;
if ( !std::getline( in, line ) )
std::getline( in, line );

// Read the first line
bool isVector = MDAL::contains( line, "VECTOR" );
group = std::make_shared< DatasetGroup >(
name(),
mesh,
mDatFile,
groupName
);
group->setIsScalar( !isVector );
group->setIsOnVertices( true );

do
{
if ( status ) *status = MDAL_Status::Err_UnknownFormat;
return;
}
line = trim( line );
// Replace tabs by spaces,
// since basement v.2.8 uses tabs instead of spaces (e.g. 'TS 0\t0.0')
line = replace( line, "\t", " " );

// http://www.xmswiki.com/xms/SMS:ASCII_Dataset_Files_*.dat
// Apart from the format specified above, there is an older supported format used in BASEMENT (and SMS?)
// which is simpler (has only one dataset in one file, no status flags etc)
bool oldFormat;
bool isVector = false;
// Trim string for cases when file has inconsistent new line symbols
line = MDAL::trim( line );

std::shared_ptr<DatasetGroup> group; // DAT outputs data
std::string groupName( MDAL::baseName( mDatFile ) );
// Split to tokens
std::vector<std::string> items = split( line, " ", SplitBehaviour::SkipEmptyParts );
if ( items.size() < 1 )
continue; // empty line?? let's skip it

if ( line == "DATASET" )
oldFormat = false;
else if ( line == "SCALAR" || line == "VECTOR" )
{
oldFormat = true;
isVector = ( line == "VECTOR" );

group = std::make_shared< DatasetGroup >(
name(),
mesh,
mDatFile,
groupName
);
group->setIsScalar( !isVector );
std::string cardType = items[0];
if ( cardType == "ND" && items.size() >= 2 )
{
size_t fileNodeCount = toSizeT( items[1] );
if ( mesh->verticesCount() != fileNodeCount )
EXIT_WITH_ERROR( MDAL_Status::Err_IncompatibleMesh );
}
else if ( cardType == "SCALAR" || cardType == "VECTOR" )
{
// just ignore - we know the type from earlier...
}
else if ( cardType == "TS" && items.size() >= 2 )
{
double t = toDouble( items[ 1 ] );
readVertexTimestep( mesh, group, t, isVector, false, in );
}
else
{
std::stringstream str;
str << " Unknown card:" << line;
debug( str.str() );
}
}
else
while ( std::getline( in, line ) );

if ( !group || group->datasets.size() == 0 )
EXIT_WITH_ERROR( MDAL_Status::Err_UnknownFormat );

group->setStatistics( MDAL::calculateStatistics( group ) );
mesh->datasetGroups.push_back( group );
group.reset();
}

void MDAL::DriverAsciiDat::loadNewFormat( std::ifstream &in,
Mesh *mesh,
MDAL_Status *status ) const
{
bool isVector = false;
std::shared_ptr<DatasetGroup> group; // DAT outputs data
std::string groupName( MDAL::baseName( mDatFile ) );
std::string line;

// see if it contains face-centered results - supported by BASEMENT
bool faceCentered = false;
if ( !oldFormat && contains( groupName, "_els_" ) )
if ( contains( groupName, "_els_" ) )
faceCentered = true;

if ( group )
Expand All @@ -127,6 +155,10 @@ void MDAL::DriverAsciiDat::load( const std::string &datFile, MDAL::Mesh *mesh, M
// since basement v.2.8 uses tabs instead of spaces (e.g. 'TS 0\t0.0')
line = replace( line, "\t", " " );

// Trim string for cases when file has inconsistent new line symbols
line = MDAL::trim( line );

// Split to tokens
std::vector<std::string> items = split( line, " ", SplitBehaviour::SkipEmptyParts );
if ( items.size() < 1 )
continue; // empty line?? let's skip it
Expand All @@ -138,18 +170,18 @@ void MDAL::DriverAsciiDat::load( const std::string &datFile, MDAL::Mesh *mesh, M
if ( mesh->verticesCount() != fileNodeCount )
EXIT_WITH_ERROR( MDAL_Status::Err_IncompatibleMesh );
}
else if ( !oldFormat && cardType == "NC" && items.size() >= 2 )
else if ( cardType == "NC" && items.size() >= 2 )
{
size_t fileElemCount = toSizeT( items[1] );
if ( mesh->facesCount() != fileElemCount )
EXIT_WITH_ERROR( MDAL_Status::Err_IncompatibleMesh );
}
else if ( !oldFormat && cardType == "OBJTYPE" )
else if ( cardType == "OBJTYPE" )
{
if ( items[1] != "mesh2d" && items[1] != "\"mesh2d\"" )
EXIT_WITH_ERROR( MDAL_Status::Err_UnknownFormat );
}
else if ( !oldFormat && ( cardType == "BEGSCL" || cardType == "BEGVEC" ) )
else if ( cardType == "BEGSCL" || cardType == "BEGVEC" )
{
if ( group )
{
Expand All @@ -167,7 +199,7 @@ void MDAL::DriverAsciiDat::load( const std::string &datFile, MDAL::Mesh *mesh, M
group->setIsScalar( !isVector );
group->setIsOnVertices( !faceCentered );
}
else if ( !oldFormat && cardType == "ENDDS" )
else if ( cardType == "ENDDS" )
{
if ( !group )
{
Expand All @@ -178,7 +210,7 @@ void MDAL::DriverAsciiDat::load( const std::string &datFile, MDAL::Mesh *mesh, M
mesh->datasetGroups.push_back( group );
group.reset();
}
else if ( !oldFormat && cardType == "NAME" && items.size() >= 2 )
else if ( cardType == "NAME" && items.size() >= 2 )
{
if ( !group )
{
Expand All @@ -191,21 +223,17 @@ void MDAL::DriverAsciiDat::load( const std::string &datFile, MDAL::Mesh *mesh, M
if ( quoteIdx1 != std::string::npos && quoteIdx2 != std::string::npos )
group->setName( line.substr( quoteIdx1 + 1, quoteIdx2 - quoteIdx1 - 1 ) );
}
else if ( oldFormat && ( cardType == "SCALAR" || cardType == "VECTOR" ) )
{
// just ignore - we know the type from earlier...
}
else if ( cardType == "TS" && items.size() >= ( oldFormat ? 2 : 3 ) )
else if ( cardType == "TS" && items.size() >= 3 )
{
double t = toDouble( items[oldFormat ? 1 : 2] );
double t = toDouble( items[2] );

if ( faceCentered )
{
readFaceTimestep( mesh, group, t, isVector, in );
}
else
{
bool hasStatus = ( oldFormat ? false : toBool( items[1] ) );
bool hasStatus = ( toBool( items[1] ) );
readVertexTimestep( mesh, group, t, isVector, hasStatus, in );
}

Expand All @@ -217,15 +245,46 @@ void MDAL::DriverAsciiDat::load( const std::string &datFile, MDAL::Mesh *mesh, M
debug( str.str() );
}
}
}

/**
* The DAT format contains "datasets" and each dataset has N-outputs. One output
* represents data for all vertices/faces for one timestep
*
* In MDAL we convert one output to one MDAL dataset;
*
*/
void MDAL::DriverAsciiDat::load( const std::string &datFile, MDAL::Mesh *mesh, MDAL_Status *status )
{
mDatFile = datFile;
if ( status ) *status = MDAL_Status::None;

if ( oldFormat )
if ( !MDAL::fileExists( mDatFile ) )
{
if ( !group || group->datasets.size() == 0 )
EXIT_WITH_ERROR( MDAL_Status::Err_UnknownFormat );
if ( status ) *status = MDAL_Status::Err_FileNotFound;
return;
}

group->setStatistics( MDAL::calculateStatistics( group ) );
mesh->datasetGroups.push_back( group );
group.reset();
std::ifstream in( mDatFile, std::ifstream::in );
std::string line;
if ( !std::getline( in, line ) )
{
if ( status ) *status = MDAL_Status::Err_UnknownFormat;
return;
}
line = trim( line );
if ( canReadNewFormat( line ) )
{
// we do not need to parse first line again
loadNewFormat( in, mesh, status );
}
else
{
// we need to parse first line again to see
// scalar/vector flag or timestep flag
in.clear();
in.seekg( 0 );
loadOldFormat( in, mesh, status );
}
}

Expand All @@ -235,7 +294,7 @@ void MDAL::DriverAsciiDat::readVertexTimestep(
double t,
bool isVector,
bool hasStatus,
std::ifstream &stream )
std::ifstream &stream ) const
{
assert( group );
size_t faceCount = mesh->facesCount();
Expand Down Expand Up @@ -301,7 +360,7 @@ void MDAL::DriverAsciiDat::readFaceTimestep(
std::shared_ptr<DatasetGroup> group,
double t,
bool isVector,
std::ifstream &stream )
std::ifstream &stream ) const
{
assert( group );

Expand Down
34 changes: 31 additions & 3 deletions external/mdal/frmts/mdal_ascii_dat.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,29 @@
namespace MDAL
{

/**
* ASCII Dat format is used by various solvers and the output
* from various solvers can have slightly different header.
* The format is used by TUFLOW, BASEMENT and HYDRO_AS-2D solvers.
*
* The most frequent form is based on official SMS documentation
* https://www.xmswiki.com/wiki/SMS:ASCII_Dataset_Files_*.dat
* The official format only supports data defined on vertices,
* both scalar vector. The new format is recognized by keyword
* "DATASET" on the first line of the file.
*
* BASEMENT solver also stores data defined on faces. To recognize
* such dataset, the dataset name contains "_els_" substring
* (e.g. depth_els_1.dat)
*
* In one file, there is always one dataset group stored.
*
* Sometime the "older" datasets may have some part of the
* header missing, e.g. the file starts with SCALAR or VECTOR or TS
* keyword. The older format does not have "active" flags for faces
* and does not recognize most of the keywords. Old format data
* are always defined on vertices
*/
class DriverAsciiDat: public Driver
{
public:
Expand All @@ -30,22 +53,27 @@ namespace MDAL
bool canRead( const std::string &uri ) override;
void load( const std::string &datFile, Mesh *mesh, MDAL_Status *status ) override;


private:
bool canReadOldFormat( const std::string &line ) const;
bool canReadNewFormat( const std::string &line ) const;

void loadOldFormat( std::ifstream &in, Mesh *mesh, MDAL_Status *status ) const;
void loadNewFormat( std::ifstream &in, Mesh *mesh, MDAL_Status *status ) const;

void readVertexTimestep(
const Mesh *mesh,
std::shared_ptr<DatasetGroup> group,
double t,
bool isVector,
bool hasStatus,
std::ifstream &stream );
std::ifstream &stream ) const;

void readFaceTimestep(
const Mesh *mesh,
std::shared_ptr<DatasetGroup> group,
double t,
bool isVector,
std::ifstream &stream );
std::ifstream &stream ) const;

std::string mDatFile;
};
Expand Down
7 changes: 4 additions & 3 deletions external/mdal/frmts/mdal_cf.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,15 @@

#include <vector>
#include <string>
#include <netcdf.h>
#include "math.h"
#include <stdlib.h>
#include <assert.h>

#include "mdal_data_model.hpp"
#include "mdal_cf.hpp"
#include "mdal_utils.hpp"

#include "math.h"
#include <stdlib.h>

#define CF_THROW_ERR throw MDAL_Status::Err_UnknownFormat

MDAL::cfdataset_info_map MDAL::DriverCF::parseDatasetGroupInfo()
Expand Down
Loading

0 comments on commit 928a559

Please sign in to comment.