Add example for TOML/JSON-based dynamic configuration

CMakeLists.txt

@@ -807,6 +807,7 @@ set(openPMD_EXAMPLE_NAMES
     10_streaming_write
     10_streaming_read
     12_span_write
+    13_write_dynamic_configuration
 )
 set(openPMD_PYTHON_EXAMPLE_NAMES
     2_read_serial
@@ -822,6 +823,7 @@ set(openPMD_PYTHON_EXAMPLE_NAMES
     10_streaming_read
     11_particle_dataframe
     12_span_write
+    13_write_dynamic_configuration
 )
 
 if(openPMD_USE_INVASIVE_TESTS)

examples/13_write_dynamic_configuration.cpp

@@ -0,0 +1,120 @@
+#include <openPMD/openPMD.hpp>
+
+#include <algorithm>
+#include <iostream>
+#include <memory>
+#include <numeric> // std::iota
+
+using std::cout;
+using namespace openPMD;
+
+int main()
+{
+    using position_t = double;
+
+    std::string const defaults = R"END(
+# This configuration is TOML-based
+# JSON can be used alternatively, the openPMD-api will automatically detect
+# the language being used
+#
+# Alternatively, the location of a JSON/TOML-file on the filesystem can
+# be passed by adding an at-sign `@` in front of the path
+# The format will then be recognized by filename extension, i.e. .json or .toml
+
+backend = "adios2"
+iteration_encoding = "group_based"
+# The following is only relevant in read mode
+defer_iteration_parsing = true
+
+[adios1.dataset]
+transform = "blosc:compressor=zlib,shuffle=bit,lvl=5;nometa"
+
+[adios2.engine]
+type = "bp4"
+
+# ADIOS2 allows adding several operators
+# Lists are given in TOML by using double brackets
+[[adios2.dataset.operators]]
+type = "zlib"
+
+parameters.clevel = 5
+# Alternatively:
+# [adios2.dataset.operators.parameters]
+# clevel = 9
+
+# For adding a further parameter:
+# [[adios2.dataset.operators]]
+# type = "some other parameter"
+# # ...
+
+[hdf5.dataset]
+chunks = "auto"
+)END";
+
+    // open file for writing
+    Series series =
+        Series( "../samples/dynamicConfig.bp", Access::CREATE, defaults );
+
+    Datatype datatype = determineDatatype< position_t >();
+    constexpr unsigned long length = 10ul;
+    Extent global_extent = { length };
+    Dataset dataset = Dataset( datatype, global_extent );
+    std::shared_ptr< position_t > local_data(
+        new position_t[ length ],
+        []( position_t const * ptr ) { delete[] ptr; } );
+
+    WriteIterations iterations = series.writeIterations();
+    for( size_t i = 0; i < 100; ++i )
+    {
+        Iteration iteration = iterations[ i ];
+        Record electronPositions = iteration.particles[ "e" ][ "position" ];
+
+        std::iota( local_data.get(), local_data.get() + length, i * length );
+        for( auto const & dim : { "x", "y", "z" } )
+        {
+            RecordComponent pos = electronPositions[ dim ];
+            pos.resetDataset( dataset );
+            pos.storeChunk( local_data, Offset{ 0 }, global_extent );
+        }
+
+        /*
+         * We want different compression settings for this dataset, so we pass
+         * a dataset-specific configuration.
+         * Also showcase how to define an resizable dataset.
+         * This time in JSON.
+         */
+        std::string const differentCompressionSettings = R"END(
+{
+  "resizable": true,
+  "adios1": {
+    "dataset": {
+      "transform": "blosc:compressor=zlib,shuffle=bit,lvl=1;nometa"
+    }
+  },
+  "adios2": {
+    "dataset": {
+      "operators": [
+        {
+          "type": "zlib",
+          "parameters": {
+            "clevel": 9
+          }
+        }
+      ]
+    }
+  }
+})END";
+        Dataset differentlyCompressedDataset{ Datatype::INT, { 10 } };
+        differentlyCompressedDataset.options = differentCompressionSettings;
+
+        auto someMesh = iteration.meshes[ "differentCompressionSettings" ]
+                                        [ RecordComponent::SCALAR ];
+        someMesh.resetDataset( differentlyCompressedDataset );
+        std::vector< int > dataVec( 10, i );
+        someMesh.storeChunk( dataVec, { 0 }, { 10 } );
+
+        iteration.close();
+    }
+
+    return 0;
+}

examples/13_write_dynamic_configuration.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python
+import json
+import openpmd_api as io
+import numpy as np
+import sys
+
+defaults = """
+# This configuration is TOML-based
+# JSON can be used alternatively, the openPMD-api will automatically detect
+# the language being used
+#
+# Alternatively, the location of a JSON/TOML-file on the filesystem can
+# be passed by adding an at-sign `@` in front of the path
+# The format will then be recognized by filename extension, i.e. .json or .toml
+
+backend = "adios2"
+iteration_encoding = "group_based"
+# The following is only relevant in read mode
+defer_iteration_parsing = true
+
+[adios1.dataset]
+transform = "blosc:compressor=zlib,shuffle=bit,lvl=5;nometa"
+
+[adios2.engine]
+type = "bp4"
+
+# ADIOS2 allows adding several operators
+# Lists are given in TOML by using double brackets
+[[adios2.dataset.operators]]
+type = "zlib"
+
+parameters.clevel = 5
+# Alternatively:
+# [adios2.dataset.operators.parameters]
+# clevel = 9
+
+# For adding a further parameter:
+# [[adios2.dataset.operators]]
+# type = "some other parameter"
+# # ...
+
+[hdf5.dataset]
+chunks = "auto"
+"""
+
+if __name__ == "__main__":
+    # create a series and specify some global metadata
+    # change the file extension to .json, .h5 or .bp for regular file writing
+    series = io.Series("../samples/dynamicConfig.bp", io.Access_Type.create,
+                       defaults)
+
+    # now, write a number of iterations (or: snapshots, time steps)
+    for i in range(10):
+        # Use `series.write_iterations()` instead of `series.iterations`
+        # for streaming support (while still retaining file-writing support).
+        # Direct access to `series.iterations` is only necessary for
+        # random-access of iterations. By using `series.write_iterations()`,
+        # the openPMD-api will adhere to streaming semantics while writing.
+        # In particular, this means that only one iteration can be written at a
+        # time and an iteration can no longer be modified after closing it.
+        iteration = series.write_iterations()[i]
+
+        #######################
+        # write electron data #
+        #######################
+
+        electronPositions = iteration.particles["e"]["position"]
+
+        # openPMD attribute
+        # (this one would also be set automatically for positions)
+        electronPositions.unit_dimension = {io.Unit_Dimension.L: 1.0}
+        # custom attribute
+        electronPositions.set_attribute("comment", "I'm a comment")
+
+        length = 10
+        local_data = np.arange(i * length, (i + 1) * length,
+                               dtype=np.dtype("double"))
+        for dim in ["x", "y", "z"]:
+            pos = electronPositions[dim]
+            pos.reset_dataset(io.Dataset(local_data.dtype, [length]))
+            pos[()] = local_data
+
+        # optionally: flush now to clear buffers
+        iteration.series_flush()  # this is a shortcut for `series.flush()`
+
+        ###############################
+        # write some temperature data #
+        ###############################
+
+        # we want different compression settings here,
+        # so we override the defaults
+        # let's use JSON this time
+        config = {
+            'resizable': True,
+            'adios2': {
+                'dataset': {
+                    'operators': []
+                }
+            },
+            'adios1': {
+                'dataset': {}
+            }
+        }
+        config['adios2']['dataset'] = {
+            'operators': [{
+                'type': 'zlib',
+                'parameters': {
+                    'clevel': 9
+                }
+            }]
+        }
+        config['adios1']['dataset'] = {
+            'transform': 'blosc:compressor=zlib,shuffle=bit,lvl=1;nometa'
+        }
+
+        temperature = iteration.meshes["temperature"]
+        temperature.unit_dimension = {io.Unit_Dimension.theta: 1.0}
+        temperature.axis_labels = ["x", "y"]
+        temperature.grid_spacing = [1., 1.]
+        # temperature has no x,y,z components, so skip the last layer:
+        temperature_dataset = temperature[io.Mesh_Record_Component.SCALAR]
+        # let's say we are in a 3x3 mesh
+        dataset = io.Dataset(np.dtype("double"), [3, 3])
+        dataset.options = json.dumps(config)
+        temperature_dataset.reset_dataset(dataset)
+        # temperature is constant
+        local_data = np.arange(i * 9, (i + 1) * 9, dtype=np.dtype("double"))
+        local_data = local_data.reshape([3, 3])
+        temperature_dataset[()] = local_data
+
+        # After closing the iteration, the readers can see the iteration.
+        # It can no longer be modified.
+        # If not closing an iteration explicitly, it will be implicitly closed
+        # upon creating the next iteration.
+        iteration.close()