add support to write and read lists from hdf5 (#554)

* working towards list support for hdf5 * working list support * docs updates and more testing * add more hdf5 list tests
LLNL · Apr 25, 2020 · 13d33e0 · 13d33e0
1 parent 647bec0
commit 13d33e0
Show file tree

Hide file tree

Showing 20 changed files with 767 additions and 400 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -20,6 +20,7 @@ and this project aspires to adhere to [Semantic Versioning](https://semver.org/s
 #### Relay
 - Added an open mode option to RelayIOHandle. See RelayIOHandle docs (https://llnl-conduit.readthedocs.io/en/latest/relay_io.html#relay-i-o-handle-interface) for more details.
 - Added the conduit.relay.mpi Python module to support Relay MPI in Python.
+- Added support to write and read Conduit lists to HDF5 files. Since HDF5 Groups do not support unnamed indexed children, each list child is written using a string name that represents its index and a special attribute is written to the HDF5 group to mark the list case. On read, the special attribute is used to detect and read this style of group back into a Conduit list.
 
 #### Blueprint
 - Added support for Adjacency sets for Structured Mesh Topologies. See the `blueprint::mesh::examples::adjset_uniform` example.

diff --git a/src/docs/sphinx/t_conduit_docs_blueprint_demos_out.txt b/src/docs/sphinx/t_conduit_docs_blueprint_demos_out.txt
@@ -31,7 +31,7 @@ fields:
 END_EXAMPLE("blueprint_demo_basic_uniform")
 [/Users/harrison37/Work/github/llnl/conduit/src/tests/docs/t_conduit_docs_blueprint_demos.cpp : 107]
  Testing Basic Example 'uniform'
-[       OK ] conduit_docs.blueprint_demo_basic_uniform (8 ms)
+[       OK ] conduit_docs.blueprint_demo_basic_uniform (9 ms)
 [ RUN      ] conduit_docs.blueprint_demo_basic_rectilinear
 BEGIN_EXAMPLE("blueprint_demo_basic_rectilinear")
 
@@ -55,7 +55,7 @@ fields:
 END_EXAMPLE("blueprint_demo_basic_rectilinear")
 [/Users/harrison37/Work/github/llnl/conduit/src/tests/docs/t_conduit_docs_blueprint_demos.cpp : 107]
  Testing Basic Example 'rectilinear'
-[       OK ] conduit_docs.blueprint_demo_basic_rectilinear (5 ms)
+[       OK ] conduit_docs.blueprint_demo_basic_rectilinear (6 ms)
 [ RUN      ] conduit_docs.blueprint_demo_basic_structured
 BEGIN_EXAMPLE("blueprint_demo_basic_structured")
 
@@ -110,7 +110,7 @@ fields:
 END_EXAMPLE("blueprint_demo_basic_tris")
 [/Users/harrison37/Work/github/llnl/conduit/src/tests/docs/t_conduit_docs_blueprint_demos.cpp : 107]
  Testing Basic Example 'tris'
-[       OK ] conduit_docs.blueprint_demo_basic_tris (6 ms)
+[       OK ] conduit_docs.blueprint_demo_basic_tris (7 ms)
 [ RUN      ] conduit_docs.blueprint_demo_basic_quads
 BEGIN_EXAMPLE("blueprint_demo_basic_quads")
 
@@ -193,7 +193,7 @@ fields:
 END_EXAMPLE("blueprint_demo_basic_hexs")
 [/Users/harrison37/Work/github/llnl/conduit/src/tests/docs/t_conduit_docs_blueprint_demos.cpp : 107]
  Testing Basic Example 'hexs'
-[       OK ] conduit_docs.blueprint_demo_basic_hexs (6 ms)
+[       OK ] conduit_docs.blueprint_demo_basic_hexs (7 ms)
 [ RUN      ] conduit_docs.blueprint_demo_basic_polygons
 BEGIN_EXAMPLE("blueprint_demo_basic_polygons")
 
@@ -220,7 +220,7 @@ fields:
 END_EXAMPLE("blueprint_demo_basic_polygons")
 [/Users/harrison37/Work/github/llnl/conduit/src/tests/docs/t_conduit_docs_blueprint_demos.cpp : 107]
  Testing Basic Example 'polygons'
-[       OK ] conduit_docs.blueprint_demo_basic_polygons (6 ms)
+[       OK ] conduit_docs.blueprint_demo_basic_polygons (7 ms)
 [ RUN      ] conduit_docs.blueprint_demo_basic_polyhedra
 BEGIN_EXAMPLE("blueprint_demo_basic_polyhedra")
 
@@ -279,9 +279,9 @@ fields:
     values: [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]
 
 END_EXAMPLE("blueprint_demo_basic_uniform_detailed")
-[       OK ] conduit_docs.blueprint_demo_basic_uniform_detailed (2 ms)
-[----------] 10 tests from conduit_docs (62 ms total)
+[       OK ] conduit_docs.blueprint_demo_basic_uniform_detailed (3 ms)
+[----------] 10 tests from conduit_docs (68 ms total)
 
 [----------] Global test environment tear-down
-[==========] 10 tests from 1 test suite ran. (62 ms total)
+[==========] 10 tests from 1 test suite ran. (68 ms total)
 [  PASSED  ] 10 tests.
diff --git a/src/docs/sphinx/t_conduit_docs_blueprint_examples_out.txt b/src/docs/sphinx/t_conduit_docs_blueprint_examples_out.txt
@@ -18,7 +18,7 @@ coords:
   valid: "true"
 
 END_EXAMPLE("blueprint_example_1")
-[       OK ] conduit_docs.blueprint_example_1 (1 ms)
+[       OK ] conduit_docs.blueprint_example_1 (2 ms)
 [ RUN      ] conduit_docs.blueprint_example_2
 BEGIN_EXAMPLE("blueprint_example_2")
 example 'separate' mcarray 
@@ -29,15 +29,15 @@ z: [3.0, 3.0, 3.0, 3.0, 3.0]
 
 
 mem_spaces: 
-  0x7fc12cc04e20: 
+  0x7fcf69404e20: 
     path: "x"
     type: "allocated"
     bytes: 40
-  0x7fc12cc04f20: 
+  0x7fcf69404f20: 
     path: "y"
     type: "allocated"
     bytes: 40
-  0x7fc12cc05080: 
+  0x7fcf69405080: 
     path: "z"
     type: "allocated"
     bytes: 40
@@ -55,7 +55,7 @@ transformed to 'interleaved' mcarray
 }
 
 mem_spaces: 
-  0x7fc12cc06b70: 
+  0x7fcf69406b70: 
     path: ""
     type: "allocated"
     bytes: 120
@@ -65,7 +65,7 @@ total_bytes_compact: 120
 total_strided_bytes: 312
 
 END_EXAMPLE("blueprint_example_2")
-[       OK ] conduit_docs.blueprint_example_2 (1 ms)
+[       OK ] conduit_docs.blueprint_example_2 (0 ms)
 [----------] 2 tests from conduit_docs (2 ms total)
 
 [----------] Global test environment tear-down

diff --git a/src/docs/sphinx/t_conduit_docs_relay_io_generic_examples_out.txt b/src/docs/sphinx/t_conduit_docs_relay_io_generic_examples_out.txt
@@ -69,7 +69,7 @@ a:
     new_data: 42.0
 
 END_EXAMPLE("relay_io_example_2_hdf5")
-[       OK ] conduit_docs.relay_io_example_2_hdf5 (3 ms)
+[       OK ] conduit_docs.relay_io_example_2_hdf5 (2 ms)
 [ RUN      ] conduit_docs.relay_io_example_3_hdf5
 BEGIN_EXAMPLE("relay_io_example_3_hdf5")
 
@@ -97,7 +97,7 @@ a:
   my_data: 1.0
 
 END_EXAMPLE("relay_io_example_3_hdf5")
-[       OK ] conduit_docs.relay_io_example_3_hdf5 (1 ms)
+[       OK ] conduit_docs.relay_io_example_3_hdf5 (2 ms)
 [ RUN      ] conduit_docs.relay_io_example_4_hdf5
 BEGIN_EXAMPLE("relay_io_example_4_hdf5")
 
@@ -113,7 +113,7 @@ Load result from 'path/to'
 my_data: 1.0
 
 END_EXAMPLE("relay_io_example_4_hdf5")
-[       OK ] conduit_docs.relay_io_example_4_hdf5 (2 ms)
+[       OK ] conduit_docs.relay_io_example_4_hdf5 (1 ms)
 [ RUN      ] conduit_docs.relay_io_example_5_hdf5
 BEGIN_EXAMPLE("relay_io_example_5_hdf5")
 
@@ -129,7 +129,7 @@ path:
     my_data: 1.0
 
 END_EXAMPLE("relay_io_example_5_hdf5")
-[       OK ] conduit_docs.relay_io_example_5_hdf5 (1 ms)
+[       OK ] conduit_docs.relay_io_example_5_hdf5 (2 ms)
 [----------] 6 tests from conduit_docs (11 ms total)
 
 [----------] Global test environment tear-down

diff --git a/src/docs/sphinx/t_conduit_docs_relay_io_handle_examples_out.txt b/src/docs/sphinx/t_conduit_docs_relay_io_handle_examples_out.txt
@@ -33,9 +33,9 @@ a:
   c: 42.0
 
 END_EXAMPLE("relay_io_handle_example_1")
-[       OK ] conduit_docs.relay_io_handle_1 (5 ms)
-[----------] 1 test from conduit_docs (5 ms total)
+[       OK ] conduit_docs.relay_io_handle_1 (6 ms)
+[----------] 1 test from conduit_docs (6 ms total)
 
 [----------] Global test environment tear-down
-[==========] 1 test from 1 test suite ran. (5 ms total)
+[==========] 1 test from 1 test suite ran. (7 ms total)
 [  PASSED  ] 1 test.
diff --git a/src/docs/sphinx/t_conduit_docs_relay_io_hdf5_examples_out.txt b/src/docs/sphinx/t_conduit_docs_relay_io_hdf5_examples_out.txt
@@ -18,13 +18,18 @@ Path 'myoutput.hdf5:a/my_data' exists
 Data loaded:
 1.0
 
+Path 'myoutput.hdf5:a/my_list/0' exists
+
 Data loaded:
 
 a: 
   my_data: 3.1415
   b: 
     my_string: "value"
     c: 144
+  my_list: 
+    - 42.0
+    - 42
 
 File 
 'myoutput.hdf5' is a hdf5 file
@@ -75,7 +80,7 @@ chunking:
 
 saving data to 'myoutput_chunked.hdf5' 
 END_EXAMPLE("relay_io_example_hdf5_interface_opts")
-[       OK ] conduit_docs.relay_io_example_hdf5_interface_2 (3 ms)
+[       OK ] conduit_docs.relay_io_example_hdf5_interface_2 (2 ms)
 [----------] 2 tests from conduit_docs (7 ms total)
 
 [----------] Global test environment tear-down

diff --git a/src/docs/sphinx/t_conduit_docs_tutorial_basics_out.txt b/src/docs/sphinx/t_conduit_docs_tutorial_basics_out.txt
@@ -36,7 +36,7 @@ list_example:
   - 4
 
 END_EXAMPLE("basics_object_and_list")
-[       OK ] conduit_tutorial.basics_object_and_list (0 ms)
+[       OK ] conduit_tutorial.basics_object_and_list (1 ms)
 [ RUN      ] conduit_tutorial.basics_object_and_list_itr
 BEGIN_EXAMPLE("basics_object_and_list_itr")
 
@@ -66,15 +66,15 @@ END_EXAMPLE("basics_object_and_list_itr")
 BEGIN_EXAMPLE("basics_mem_spaces")
 
 mem_spaces: 
-  0x7f9239c04e40: 
+  0x7fa457501230: 
     path: "my"
     type: "allocated"
     bytes: 5
-  0x7f9239c05420: 
+  0x7fa457501810: 
     path: "a/b/c"
     type: "allocated"
     bytes: 2
-  0x7f9239c05e50: 
+  0x7fa457502240: 
     path: "a/b/e"
     type: "allocated"
     bytes: 8
@@ -104,7 +104,7 @@ BEGIN_EXAMPLE("basics_bw_style_from_native")
 }
 END_EXAMPLE("basics_bw_style_from_native")
 [       OK ] conduit_tutorial.basics_bw_style_from_native (0 ms)
-[----------] 7 tests from conduit_tutorial (0 ms total)
+[----------] 7 tests from conduit_tutorial (1 ms total)
 
 [----------] Global test environment tear-down
 [==========] 7 tests from 1 test suite ran. (1 ms total)

diff --git a/src/docs/sphinx/t_conduit_docs_tutorial_errors_out.txt b/src/docs/sphinx/t_conduit_docs_tutorial_errors_out.txt
@@ -16,9 +16,9 @@ BEGIN_EXAMPLE("error_handlers_rewire")
 END_EXAMPLE("error_handlers_rewire")
 BEGIN_EXAMPLE("error_handlers_reset")
 END_EXAMPLE("error_handlers_reset")
-[       OK ] conduit_tutorial.error_handlers (0 ms)
-[----------] 3 tests from conduit_tutorial (0 ms total)
+[       OK ] conduit_tutorial.error_handlers (1 ms)
+[----------] 3 tests from conduit_tutorial (1 ms total)
 
 [----------] Global test environment tear-down
-[==========] 3 tests from 1 test suite ran. (0 ms total)
+[==========] 3 tests from 1 test suite ran. (1 ms total)
 [  PASSED  ] 3 tests.
diff --git a/src/docs/sphinx/t_conduit_docs_tutorial_numeric_out.txt b/src/docs/sphinx/t_conduit_docs_tutorial_numeric_out.txt
@@ -53,12 +53,12 @@ vals[0] = 1000
 vals[1] = 2000
 vals[2] = 3000
 vals[3] = 4000
-[       OK ] conduit_tutorial.numeric_double_conversion (1 ms)
+[       OK ] conduit_tutorial.numeric_double_conversion (0 ms)
 [ RUN      ] conduit_tutorial.numeric_double_conversion_end
 END_EXAMPLE("numeric_double_conversion")
 [       OK ] conduit_tutorial.numeric_double_conversion_end (0 ms)
-[----------] 8 tests from conduit_tutorial (1 ms total)
+[----------] 8 tests from conduit_tutorial (0 ms total)
 
 [----------] Global test environment tear-down
-[==========] 8 tests from 1 test suite ran. (1 ms total)
+[==========] 8 tests from 1 test suite ran. (0 ms total)
 [  PASSED  ] 8 tests.
diff --git a/src/docs/sphinx/t_conduit_docs_tutorial_ownership_out.txt b/src/docs/sphinx/t_conduit_docs_tutorial_ownership_out.txt
@@ -6,11 +6,11 @@ Running main() from /Users/harrison37/Work/github/llnl/conduit/src/blt/thirdpart
 BEGIN_EXAMPLE("mem_ownership_external")
 
 mem_spaces: 
-  0x7fd547c04c50: 
+  0x7f9fe6c04c50: 
     path: "v_owned"
     type: "allocated"
     bytes: 40
-  0x7fd547c04890: 
+  0x7f9fe6c04890: 
     path: "v_external"
     type: "external"
 total_bytes_allocated: 40

diff --git a/src/docs/sphinx/t_conduit_docs_tutorial_parse_out.txt b/src/docs/sphinx/t_conduit_docs_tutorial_parse_out.txt
@@ -19,7 +19,7 @@ BEGIN_EXAMPLE("t_conduit_docs_tutorial_json")
   "mykey": {"dtype":"float64", "number_of_elements": 1, "offset": 0, "stride": 8, "element_bytes": 8, "endianness": "little", "value": 42.0}
 }
 END_EXAMPLE("t_conduit_docs_tutorial_json")
-[       OK ] conduit_tutorial.t_conduit_docs_tutorial_json (0 ms)
+[       OK ] conduit_tutorial.t_conduit_docs_tutorial_json (1 ms)
 [ RUN      ] conduit_tutorial.t_conduit_docs_tutorial_yaml_inline_array
 BEGIN_EXAMPLE("t_conduit_docs_tutorial_yaml_inline_array")
 [0.0, 10.0, 20.0, 30.0]
@@ -75,7 +75,7 @@ b: 200.0
 
 
 mem_spaces: 
-  0x7ffeea50a030: 
+  0x7ffee952f030: 
     path: "a"
     type: "external"
 total_bytes_allocated: 0
@@ -84,14 +84,14 @@ total_bytes_compact: 16
 total_strided_bytes: 16
 
 END_EXAMPLE("t_json_generator_bind_to_incore")
-[       OK ] conduit_tutorial.json_generator_bind_to_incore (1 ms)
+[       OK ] conduit_tutorial.json_generator_bind_to_incore (0 ms)
 [ RUN      ] conduit_tutorial.json_generator_compact
 BEGIN_EXAMPLE("t_json_generator_compact")
 [100.0, 200.0, 300.0, 400.0, 500.0]
 [-100.0, -200.0, -300.0, -400.0, -500.0]
 
 mem_spaces: 
-  0x7ffeea509ff0: 
+  0x7ffee952eff0: 
     path: ""
     type: "external"
 total_bytes_allocated: 0
@@ -101,7 +101,7 @@ total_strided_bytes: 72
 
 
 mem_spaces: 
-  0x7ffeea509ff0: 
+  0x7ffee952eff0: 
     path: ""
     type: "external"
 total_bytes_allocated: 0
@@ -113,7 +113,7 @@ total_strided_bytes: 72
 {"dtype":"float64", "number_of_elements": 5, "offset": 0, "stride": 8, "element_bytes": 8, "endianness": "little"}
 
 mem_spaces: 
-  0x7fa1db4056e0: 
+  0x7feb244056e0: 
     path: ""
     type: "allocated"
     bytes: 40
@@ -125,7 +125,7 @@ total_strided_bytes: 40
 [-100.0, -200.0, -300.0, -400.0, -500.0]
 
 mem_spaces: 
-  0x7fa1db406350: 
+  0x7feb24406350: 
     path: ""
     type: "allocated"
     bytes: 40
@@ -135,9 +135,9 @@ total_bytes_compact: 40
 total_strided_bytes: 40
 
 END_EXAMPLE("t_json_generator_compact")
-[       OK ] conduit_tutorial.json_generator_compact (0 ms)
-[----------] 8 tests from conduit_tutorial (1 ms total)
+[       OK ] conduit_tutorial.json_generator_compact (1 ms)
+[----------] 8 tests from conduit_tutorial (2 ms total)
 
 [----------] Global test environment tear-down
-[==========] 8 tests from 1 test suite ran. (1 ms total)
+[==========] 8 tests from 1 test suite ran. (2 ms total)
 [  PASSED  ] 8 tests.
diff --git a/src/docs/sphinx/t_conduit_docs_tutorial_python_basics_out.txt b/src/docs/sphinx/t_conduit_docs_tutorial_python_basics_out.txt
@@ -25,7 +25,7 @@ test 106
 106: test_007_basics_bw_style_from_native (t_conduit_docs_tutorial_python_basics.Conduit_Tutorial_Python_Basics) ... ok
 106: 
 106: ----------------------------------------------------------------------
-106: Ran 7 tests in 0.001s
+106: Ran 7 tests in 0.003s
 106: 
 106: OK
 106: BEGIN_EXAMPLE("py_basics_very_basic")
@@ -84,15 +84,15 @@ test 106
 106: BEGIN_EXAMPLE("py_basics_mem_spaces")
 106: 
 106: mem_spaces: 
-106:   0x7ff41ae58e90: 
+106:   0x7fc64573faa0: 
 106:     path: "my"
 106:     type: "allocated"
 106:     bytes: 5
-106:   0x7ff41ae84120: 
+106:   0x7fc645745590: 
 106:     path: "a/b/c"
 106:     type: "allocated"
 106:     bytes: 2
-106:   0x7ff41ae7d8a0: 
+106:   0x7fc645741e50: 
 106:     path: "a/b/e"
 106:     type: "allocated"
 106:     bytes: 8
@@ -113,11 +113,11 @@ test 106
 106:   "test": {"dtype":"int64", "number_of_elements": 1, "offset": 0, "stride": 8, "element_bytes": 8, "endianness": "little"}
 106: }
 106: END_EXAMPLE("py_basics_bw_style_from_native")
-1/1 Test #106: t_conduit_docs_tutorial_python_basics ...   Passed    0.15 sec
+1/1 Test #106: t_conduit_docs_tutorial_python_basics ...   Passed    0.34 sec
 
 The following tests passed:
 	t_conduit_docs_tutorial_python_basics
 
 100% tests passed, 0 tests failed out of 1
 
-Total Test time (real) =   0.17 sec
+Total Test time (real) =   0.40 sec