time handling (#179)

Introduce DateTime and RelativeTimestamp. Implemented reading of the reference/relative time for all formats we have testcases for. Changed API that it always returns UTF datetime. 

fixes #177

mdal/CMakeLists.txt

@@ -6,6 +6,7 @@ SET(MDAL_SOURCES
   mdal_utils.cpp
   mdal_driver_manager.cpp
   mdal_data_model.cpp
+  mdal_datetime.cpp
   mdal_memory_data_model.cpp
   frmts/mdal_driver.cpp
   frmts/mdal_2dm.cpp
@@ -20,6 +21,7 @@ SET(MDAL_HEADERS
   mdal_utils.hpp
   mdal_driver_manager.hpp
   mdal_data_model.hpp
+  mdal_datetime.hpp
   mdal_memory_data_model.hpp
   frmts/mdal_driver.hpp
   frmts/mdal_2dm.hpp

mdal/api/mdal.h

@@ -298,7 +298,7 @@ MDAL_EXPORT void MDAL_G_minimumMaximum( DatasetGroupH group, double *min, double
 //! Only for 2D datasets
 //!
 //! \param group parent group handle
-//! \param time time for dataset
+//! \param time time for dataset (hours)
 //! \param values For scalar data on vertices, the size must be vertex count
 //!               For scalar data on faces, the size must be faces count
 //!               For vector data on vertices, the size must be vertex count * 2 (x1, y1, x2, y2, ..., xN, yN)
@@ -321,8 +321,7 @@ MDAL_EXPORT bool MDAL_G_isInEditMode( DatasetGroupH group );
 //! When closed, minimum and maximum dataset group values are automatically calculated
 MDAL_EXPORT void MDAL_G_closeEditMode( DatasetGroupH group );
 
-//! Returns reference time for dataset group
-//! If returned value begins with word JULIAN, following number represents date in Julian format
+//! Returns reference time for dataset group expressed in date with ISO8601 format, return "" if reference time is not defined
 MDAL_EXPORT const char *MDAL_G_referenceTime( DatasetGroupH group );
 
 ///////////////////////////////////////////////////////////////////////////////////////
@@ -332,7 +331,7 @@ MDAL_EXPORT const char *MDAL_G_referenceTime( DatasetGroupH group );
 //! Returns dataset parent group
 MDAL_EXPORT DatasetGroupH MDAL_D_group( DatasetH dataset );
 
-//! Returns dataset time
+//! Returns dataset time (hours)
 MDAL_EXPORT double MDAL_D_time( DatasetH dataset );
 
 //! Returns volumes count for the mesh (for 3D meshes)

mdal/frmts/mdal_3di.cpp

@@ -139,7 +139,7 @@ void MDAL::Driver3Di::addBedElevation( MemoryMesh *mesh )
   group->setIsScalar( true );
 
   std::shared_ptr<MDAL::MemoryDataset2D> dataset = std::make_shared< MemoryDataset2D >( group.get() );
-  dataset->setTime( 0.0 );
+  dataset->setTime( MDAL::RelativeTimestamp() );
   for ( size_t i = 0; i < faceCount; ++i )
   {
     dataset->setScalarValue( i, MDAL::safeValue( coordZ[i], fillZ ) );

mdal/frmts/mdal_ascii_dat.cpp

@@ -116,7 +116,8 @@ void MDAL::DriverAsciiDat::loadOldFormat( std::ifstream &in,
     }
     else if ( cardType == "TS" && items.size() >=  2 )
     {
-      double t = toDouble( items[ 1 ] );
+      double rawTime = toDouble( items[ 1 ] );
+      MDAL::RelativeTimestamp t( rawTime, MDAL::RelativeTimestamp::hours );
       readVertexTimestep( mesh, group, t, isVector, false, in );
     }
     else
@@ -147,7 +148,7 @@ void MDAL::DriverAsciiDat::loadNewFormat(
   std::shared_ptr<DatasetGroup> group; // DAT outputs data
   std::string groupName( MDAL::baseName( mDatFile ) );
   std::string line;
-  std::string referenceTime;
+  MDAL::DateTime referenceTime;
   // see if it contains face-centered results - supported by BASEMENT
   bool faceCentered = false;
   if ( contains( groupName, "_els" ) )
@@ -231,7 +232,7 @@ void MDAL::DriverAsciiDat::loadNewFormat(
     }
     else if ( cardType == "RT_JULIAN" && items.size() >= 2 )
     {
-      referenceTime = "JULIAN " + items[1];
+      referenceTime = DateTime( MDAL::toDouble( items[1] ), DateTime::JulianDay );
     }
     else if ( cardType == "TIMEUNITS" && items.size() >= 2 )
     {
@@ -245,8 +246,8 @@ void MDAL::DriverAsciiDat::loadNewFormat(
     }
     else if ( cardType == "TS" && items.size() >= 3 )
     {
-      double t = toDouble( items[2] );
-      t = convertTimeDataToHours( t, group->getMetadata( "TIMEUNITS" ) );
+      double rawTime = toDouble( items[2] );
+      MDAL::RelativeTimestamp t( rawTime, MDAL::parseDurationTimeUnit( group->getMetadata( "TIMEUNITS" ) ) );
 
       if ( faceCentered )
       {
@@ -329,7 +330,7 @@ void MDAL::DriverAsciiDat::load( const std::string &datFile, MDAL::Mesh *mesh, M
 void MDAL::DriverAsciiDat::readVertexTimestep(
   const MDAL::Mesh *mesh,
   std::shared_ptr<DatasetGroup> group,
-  double t,
+  MDAL::RelativeTimestamp t,
   bool isVector,
   bool hasStatus,
   std::ifstream &stream ) const
@@ -398,7 +399,7 @@ void MDAL::DriverAsciiDat::readVertexTimestep(
 void MDAL::DriverAsciiDat::readFaceTimestep(
   const MDAL::Mesh *mesh,
   std::shared_ptr<DatasetGroup> group,
-  double t,
+  MDAL::RelativeTimestamp t,
   bool isVector,
   std::ifstream &stream ) const
 {
@@ -477,17 +478,11 @@ bool MDAL::DriverAsciiDat::persist( MDAL::DatasetGroup *group )
   out << "ND " << nodeCount << "\n";
   out << "NC " << elemCount << "\n";
   out << "NAME " "\"" << group->name() << "\"" "\n";
-  std::string referenceTimeStr = group->referenceTime();
+  std::string referenceTimeStr = group->referenceTime().toJulianDayString();
 
   if ( !referenceTimeStr.empty() )
   {
-    // Cutting of the JULIAN prefix
-    std::vector<std::string> referenceTimeStrWords = split( referenceTimeStr,  ' ' );
-
-    if ( referenceTimeStrWords.size() > 1 )
-      out << "RT_JULIAN " << referenceTimeStrWords[1] << "\n";
-    else
-      out << "RT_JULIAN " << referenceTimeStr << "\n";
+    out << "RT_JULIAN " << referenceTimeStr << "\n";
   }
 
   out << "TIMEUNITS " << 0 << "\n";
@@ -498,7 +493,7 @@ bool MDAL::DriverAsciiDat::persist( MDAL::DatasetGroup *group )
       = std::dynamic_pointer_cast<MDAL::MemoryDataset2D>( group->datasets[time_index] );
 
     bool hasActiveStatus = isOnVertices && dataset->supportsActiveFlag();
-    out << "TS " << hasActiveStatus << " " << std::to_string( dataset->time() ) << "\n";
+    out << "TS " << hasActiveStatus << " " << std::to_string( dataset->time( RelativeTimestamp::hours ) ) << "\n";
 
     if ( hasActiveStatus )
     {
@@ -528,21 +523,3 @@ bool MDAL::DriverAsciiDat::persist( MDAL::DatasetGroup *group )
 
   return false;
 }
-
-double MDAL::DriverAsciiDat::convertTimeDataToHours( double time, const std::string &originalTimeDataUnit ) const
-{
-  if ( originalTimeDataUnit == "se" || originalTimeDataUnit == "2" || originalTimeDataUnit == "Seconds"
-       || originalTimeDataUnit.empty() )
-  {
-    time /= 3600.0;
-  }
-  else if ( originalTimeDataUnit == "mi" || originalTimeDataUnit == "1" || originalTimeDataUnit == "Minutes" )
-  {
-    time /= 60.0;
-  }
-  else if ( originalTimeDataUnit == "days" )
-  {
-    time *= 24;
-  }
-  return time;
-}

mdal/frmts/mdal_ascii_dat.hpp

@@ -70,22 +70,18 @@ namespace MDAL
       //! maximum native index of the vertex in defined in the mesh
       size_t maximumId( const Mesh *mesh ) const;
 
-      void readVertexTimestep(
-        const Mesh *mesh,
-        std::shared_ptr<DatasetGroup> group,
-        double t,
-        bool isVector,
-        bool hasStatus,
-        std::ifstream &stream ) const;
+      void readVertexTimestep( const Mesh *mesh,
+                               std::shared_ptr<DatasetGroup> group,
+                               RelativeTimestamp t,
+                               bool isVector,
+                               bool hasStatus,
+                               std::ifstream &stream ) const;
 
-      void readFaceTimestep(
-        const Mesh *mesh,
-        std::shared_ptr<DatasetGroup> group,
-        double t,
-        bool isVector,
-        std::ifstream &stream ) const;
-
-      double convertTimeDataToHours( double time, const std::string &originalTimeDataUnit ) const;
+      void readFaceTimestep( const Mesh *mesh,
+                             std::shared_ptr<DatasetGroup> group,
+                             RelativeTimestamp t,
+                             bool isVector,
+                             std::ifstream &stream ) const;
 
       std::string mDatFile;
   };

mdal/frmts/mdal_binary_dat.cpp

@@ -251,7 +251,7 @@ void MDAL::DriverBinaryDat::load( const std::string &datFile, MDAL::Mesh *mesh,
           return exit_with_error( status, MDAL_Status::Err_UnknownFormat, "unable to read reference time" );
 
         referenceTime = static_cast<double>( time );
-        group->setReferenceTime( "JULIAN " + std::to_string( referenceTime ) );
+        group->setReferenceTime( DateTime( referenceTime, DateTime::JulianDay ) );
         break;
 
       case CT_TIMEUNITS:
@@ -288,8 +288,8 @@ void MDAL::DriverBinaryDat::load( const std::string &datFile, MDAL::Mesh *mesh,
         if ( read( in, reinterpret_cast< char * >( &time ), 4 ) )
           return exit_with_error( status, MDAL_Status::Err_UnknownFormat, "Invalid time step" );
 
-        double t = static_cast<double>( time );
-        t = convertTimeDataToHours( t, timeUnit );
+        double rawTime = static_cast<double>( time );
+        MDAL::RelativeTimestamp t( rawTime, MDAL::parseDurationTimeUnit( timeUnitStr ) );
 
         if ( readVertexTimestep( mesh, group, groupMax, t, istat, sflg, in ) )
           return exit_with_error( status, MDAL_Status::Err_UnknownFormat, "Unable to read vertex timestep" );
@@ -315,7 +315,7 @@ bool MDAL::DriverBinaryDat::readVertexTimestep(
   const MDAL::Mesh *mesh,
   std::shared_ptr<DatasetGroup> group,
   std::shared_ptr<DatasetGroup> groupMax,
-  double time,
+  MDAL::RelativeTimestamp time,
   bool hasStatus,
   int sflg,
   std::ifstream &in )
@@ -364,15 +364,15 @@ bool MDAL::DriverBinaryDat::readVertexTimestep(
     }
   }
 
-  if ( MDAL::equals( time, 99999.0 ) ) // Special TUFLOW dataset with maximus
+  if ( MDAL::equals( time.value( MDAL::RelativeTimestamp::hours ), 99999.0 ) ) // Special TUFLOW dataset with maximus
   {
     dataset->setTime( time );
     dataset->setStatistics( MDAL::calculateStatistics( dataset ) );
     groupMax->datasets.push_back( dataset );
   }
   else
   {
-    dataset->setTime( time ); // TODO read TIMEUNITS
+    dataset->setTime( time );
     dataset->setStatistics( MDAL::calculateStatistics( dataset ) );
     group->datasets.push_back( dataset );
   }
@@ -458,7 +458,7 @@ bool MDAL::DriverBinaryDat::persist( MDAL::DatasetGroup *group )
 
     writeRawData( out, reinterpret_cast< const char * >( &CT_TS ), 4 );
     writeRawData( out, reinterpret_cast< const char * >( &istat ), 1 );
-    float ftime = static_cast<float>( dataset->time() );
+    float ftime = static_cast<float>( dataset->time( RelativeTimestamp::hours ) );
     writeRawData( out, reinterpret_cast< const char * >( &ftime ), 4 );
 
     if ( istat )
@@ -493,23 +493,3 @@ bool MDAL::DriverBinaryDat::persist( MDAL::DatasetGroup *group )
 
   return false;
 }
-
-double MDAL::DriverBinaryDat::convertTimeDataToHours( double time, int originalTimeDataUnit )
-{
-  switch ( originalTimeDataUnit )
-  {
-    case 1:
-      time /= 60.0;
-      break;
-    case 2:
-      time /= 3600.0;
-      break;
-    case 4:
-      time *= 24;
-      break;
-    case 0:
-    default:
-      break;
-  }
-  return time;
-}

mdal/frmts/mdal_binary_dat.hpp

@@ -35,12 +35,11 @@ namespace MDAL
       bool readVertexTimestep( const Mesh *mesh,
                                std::shared_ptr<DatasetGroup> group,
                                std::shared_ptr<DatasetGroup> groupMax,
-                               double time,
+                               RelativeTimestamp time,
                                bool hasStatus,
                                int sflg,
                                std::ifstream &in );
 
-      double convertTimeDataToHours( double time, int originalTimeDataUnit );
       std::string mDatFile;
   };
 

mdal/frmts/mdal_cf.cpp

@@ -164,7 +164,7 @@ static void populate_vals( bool is_vector, double *vals, size_t i,
   }
 }
 
-void MDAL::DriverCF::addDatasetGroups( MDAL::Mesh *mesh, const std::vector<double> &times, const MDAL::cfdataset_info_map &dsinfo_map )
+void MDAL::DriverCF::addDatasetGroups( MDAL::Mesh *mesh, const std::vector<RelativeTimestamp> &times, const MDAL::cfdataset_info_map &dsinfo_map, const MDAL::DateTime &referenceTime )
 {
   /* PHASE 2 - add dataset groups */
   for ( const auto &it : dsinfo_map )
@@ -206,7 +206,6 @@ void MDAL::DriverCF::addDatasetGroups( MDAL::Mesh *mesh, const std::vector<doubl
     for ( size_t ts = 0; ts < dsi.nTimesteps; ++ts )
     {
       std::shared_ptr<MDAL::Dataset> dataset;
-      double time = times[ts];
       if ( dsi.outputType == CFDimensions::Volume3D )
       {
         dataset = create3DDataset(
@@ -223,7 +222,7 @@ void MDAL::DriverCF::addDatasetGroups( MDAL::Mesh *mesh, const std::vector<doubl
 
       if ( dataset )
       {
-        dataset->setTime( time );
+        dataset->setTime( times[ts] );
         group->datasets.push_back( dataset );
       }
     }
@@ -232,30 +231,38 @@ void MDAL::DriverCF::addDatasetGroups( MDAL::Mesh *mesh, const std::vector<doubl
     if ( !group->datasets.empty() )
     {
       group->setStatistics( MDAL::calculateStatistics( group ) );
+      group->setReferenceTime( referenceTime );
       mesh->datasetGroups.push_back( group );
     }
   }
 }
 
-void MDAL::DriverCF::parseTime( std::vector<double> &times )
+MDAL::DateTime MDAL::DriverCF::parseTime( std::vector<RelativeTimestamp> &times )
 {
 
   size_t nTimesteps = mDimensions.size( CFDimensions::Time );
   if ( 0 == nTimesteps )
   {
     //if no time dimension is present creates only one time step to store the potential time-independent variable
     nTimesteps = 1;
-    times = std::vector<double>( 1, 0 );
-    return;
+    times = std::vector<RelativeTimestamp>( 1, RelativeTimestamp() );
+    return MDAL::DateTime();
   }
   const std::string timeArrName = getTimeVariableName();
-  times = mNcFile->readDoubleArr( timeArrName, nTimesteps );
-  std::string units = mNcFile->getAttrStr( timeArrName, "units" );
-  double div_by = MDAL::parseTimeUnits( units );
+  std::vector<double> rawTimes = mNcFile->readDoubleArr( timeArrName, nTimesteps );
+
+  std::string timeUnitInformation = mNcFile->getAttrStr( timeArrName, "units" );
+  std::string calendar = mNcFile->getAttrStr( timeArrName, "calendar" );
+  MDAL::DateTime referenceTime = parseCFReferenceTime( timeUnitInformation, calendar );
+  MDAL::RelativeTimestamp::Unit unit = parseCFTimeUnit( timeUnitInformation );
+
+  times = std::vector<RelativeTimestamp>( nTimesteps );
   for ( size_t i = 0; i < nTimesteps; ++i )
   {
-    times[i] /= div_by;
+    times[i] = RelativeTimestamp( rawTimes[i], unit );
   }
+
+  return referenceTime;
 }
 
 std::shared_ptr<MDAL::Dataset> MDAL::DriverCF::create2DDataset( std::shared_ptr<MDAL::DatasetGroup> group, size_t ts, const MDAL::CFDatasetGroupInfo &dsi, double fill_val_x, double fill_val_y )
@@ -371,7 +378,7 @@ std::unique_ptr< MDAL::Mesh > MDAL::DriverCF::load( const std::string &fileName,
   if ( status ) *status = MDAL_Status::None;
 
   //Dimensions dims;
-  std::vector<double> times;
+  std::vector<MDAL::RelativeTimestamp> times;
 
   try
   {
@@ -401,13 +408,13 @@ std::unique_ptr< MDAL::Mesh > MDAL::DriverCF::load( const std::string &fileName,
     setProjection( mesh.get() );
 
     // Parse time array
-    parseTime( times );
+    MDAL::DateTime referenceTime = parseTime( times );
 
     // Parse dataset info
     cfdataset_info_map dsinfo_map = parseDatasetGroupInfo();
 
     // Create datasets
-    addDatasetGroups( mesh.get(), times, dsinfo_map );
+    addDatasetGroups( mesh.get(), times, dsinfo_map, referenceTime );
 
     return std::unique_ptr<Mesh>( mesh.release() );
   }

mdal/frmts/mdal_cf.hpp

@@ -138,10 +138,10 @@ namespace MDAL
 
       void setProjection( MDAL::Mesh *m );
       cfdataset_info_map parseDatasetGroupInfo();
-      void parseTime( std::vector<double> &times );
+      DateTime parseTime( std::vector<RelativeTimestamp> &times ); //Return the reference time
       void addDatasetGroups( Mesh *mesh,
-                             const std::vector<double> &times,
-                             const cfdataset_info_map &dsinfo_map );
+                             const std::vector<RelativeTimestamp> &times,
+                             const cfdataset_info_map &dsinfo_map, const DateTime &referenceTime );
 
       std::string mFileName;
       std::shared_ptr<NetCDFFile> mNcFile;