From 0e723f12e93eeb999481f59e85e792658b0b6873 Mon Sep 17 00:00:00 2001 From: Cyrus Harrison Date: Sun, 19 Apr 2020 17:09:29 -0700 Subject: [PATCH 1/5] py: add initial conduit.relay.mpi module and tests (#542) * py: add initial conduit.relay.mpi module and tests * add docs and example python script for relay mpi * update changelog * relay mpi: skip py mpi tests on windows until we can get a sane ci setup --- CHANGELOG.md | 1 + appveyor.yml | 5 +- scripts/uberenv/packages/conduit/package.py | 498 ++++ scripts/uberenv/project.json | 1 + src/cmake/SetupTests.cmake | 72 +- src/docs/sphinx/relay_mpi.rst | 147 ++ ...python_tutorial_relay_mpi_examples_out.txt | 116 + src/libs/relay/python/CMakeLists.txt | 22 + .../relay/python/conduit_relay_mpi_python.cpp | 2137 +++++++++++++++++ .../python/conduit_relay_python_exports.h.in | 2 + src/libs/relay/python/py_src/__init__.py | 1 + src/libs/relay/python/py_src/mpi/__init__.py | 53 + src/libs/relay/python/setup.py | 1 + src/tests/docs/CMakeLists.txt | 12 + ...docs_tutorial_python_relay_mpi_examples.py | 299 +++ src/tests/relay/python/CMakeLists.txt | 9 + src/tests/relay/python/t_python_relay_mpi.py | 430 ++++ 17 files changed, 3803 insertions(+), 3 deletions(-) create mode 100644 scripts/uberenv/packages/conduit/package.py create mode 100644 src/docs/sphinx/t_conduit_docs_python_tutorial_relay_mpi_examples_out.txt create mode 100644 src/libs/relay/python/conduit_relay_mpi_python.cpp create mode 100644 src/libs/relay/python/py_src/mpi/__init__.py create mode 100644 src/tests/docs/t_conduit_docs_tutorial_python_relay_mpi_examples.py create mode 100644 src/tests/relay/python/t_python_relay_mpi.py diff --git a/CHANGELOG.md b/CHANGELOG.md index fb1e866f7..292799f4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ and this project aspires to adhere to [Semantic Versioning](https://semver.org/s #### Relay - Added an open mode option to RelayIOHandle. See RelayIOHandle docs (https://llnl-conduit.readthedocs.io/en/latest/relay_io.html#relay-i-o-handle-interface) for more details. +- Added the conduit.relay.mpi Python module to support Relay MPI in Python. ### Fixed diff --git a/appveyor.yml b/appveyor.yml index 92e1da3ca..63c1f8d76 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -56,14 +56,17 @@ install: - conda install numpy # try to import numpy to as conda gut-check - python -c "import numpy as n; print(n.__version__); print(n.get_include());" + # try to import numpy to as conda gut-check # Install MS-MPI - ps: Start-FileDownload 'https://download.microsoft.com/download/B/2/E/B2EB83FE-98C2-4156-834A-E1711E6884FB/MSMpiSetup.exe' - MSMpiSetup.exe -unattend - set PATH=C:\Program Files\Microsoft MPI\Bin;%PATH% - # Install MS-MPI SDK - ps: Start-FileDownload 'https://download.microsoft.com/download/B/2/E/B2EB83FE-98C2-4156-834A-E1711E6884FB/msmpisdk.msi' - msmpisdk.msi /passive + - set PATH=C:\Program Files (x86)\Microsoft SDKs\MPI;%PATH% + # install mpi4py + - pip install mpi4py # Install CMake 3.9 ############################################################################ diff --git a/scripts/uberenv/packages/conduit/package.py b/scripts/uberenv/packages/conduit/package.py new file mode 100644 index 000000000..084e390ee --- /dev/null +++ b/scripts/uberenv/packages/conduit/package.py @@ -0,0 +1,498 @@ +# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack import * + +import socket +import os +import glob +import shutil + +import llnl.util.tty as tty +from os import environ as env + + +def cmake_cache_entry(name, value, vtype=None): + """ + Helper that creates CMake cache entry strings used in + 'host-config' files. + """ + if vtype is None: + if value == "ON" or value == "OFF": + vtype = "BOOL" + else: + vtype = "PATH" + return 'set({0} "{1}" CACHE {2} "")\n\n'.format(name, value, vtype) + + +class Conduit(Package): + """Conduit is an open source project from Lawrence Livermore National + Laboratory that provides an intuitive model for describing hierarchical + scientific data in C++, C, Fortran, and Python. It is used for data + coupling between packages in-core, serialization, and I/O tasks.""" + + homepage = "http://software.llnl.gov/conduit" + url = "https://github.com/LLNL/conduit/releases/download/v0.3.0/conduit-v0.3.0-src-with-blt.tar.gz" + git = "https://github.com/LLNL/conduit.git" + + version('master', branch='master', submodules=True, preferred=True) + version('0.4.0', sha256='c228e6f0ce5a9c0ffb98e0b3d886f2758ace1a4b40d00f3f118542c0747c1f52') + version('0.3.1', sha256='7b358ca03bb179876291d4a55d6a1c944b7407a80a588795b9e47940b1990521') + version('0.3.0', sha256='52e9cf5720560e5f7492876c39ef2ea20ae73187338361d2744bdf67567da155') + # note: checksums on github automatic release source tars changed ~9/17 + version('0.2.1', sha256='796576b9c69717c52f0035542c260eb7567aa351ee892d3fbe3521c38f1520c4') + version('0.2.0', sha256='31eff8dbc654a4b235cfcbc326a319e1752728684296721535c7ca1c9b463061') + + maintainers = ['cyrush'] + + ########################################################################### + # package variants + ########################################################################### + + variant("shared", default=True, description="Build Conduit as shared libs") + variant('test', default=True, description='Enable Conduit unit tests') + + # variants for python support + variant("python", default=True, description="Build Conduit Python support") + variant("fortran", default=True, description="Build Conduit Fortran support") + + # variants for comm and i/o + variant("mpi", default=True, description="Build Conduit MPI Support") + variant("hdf5", default=True, description="Build Conduit HDF5 support") + variant("hdf5_compat", default=True, + description="Build Conduit with HDF5 1.8.x (compatibility mode)") + variant("silo", default=False, description="Build Conduit Silo support") + variant("adios", default=False, description="Build Conduit ADIOS support") + + # zfp compression + variant("zfp", default=True, description="Build Conduit ZFP support") + + # variants for dev-tools (docs, etc) + variant("doc", default=False, description="Build Conduit's documentation") + # doxygen support is wip, since doxygen has several dependencies + # we want folks to explicitly opt in to building doxygen + variant("doxygen", default=False, description="Build Conduit's Doxygen documentation") + + ########################################################################### + # package dependencies + ########################################################################### + + ####################### + # CMake + ####################### + # cmake 3.8.2 or newer + depends_on("cmake@3.8.2:", type='build') + + ####################### + # Python + ####################### + # we need a shared version of python b/c linking with static python lib + # causes duplicate state issues when running compiled python modules. + depends_on("python", when="+python") + extends("python", when="+python") + depends_on("py-numpy", when="+python", type=('build', 'run')) + + ####################### + # I/O Packages + ####################### + + ############### + # HDF5 + ############### + # TODO: cxx variant is disabled due to build issue Cyrus + # experienced on BGQ. When on, the static build tries + # to link against shared libs. + # + # Use HDF5 1.8, for wider output compatibly + # variants reflect we are not using hdf5's mpi or fortran features. + depends_on("hdf5@1.8.19:1.8.999~cxx~mpi~fortran", when="+hdf5+hdf5_compat+shared") + depends_on("hdf5@1.8.19:1.8.999~shared~cxx~mpi~fortran", when="+hdf5+hdf5_compat~shared") + depends_on("hdf5~cxx~mpi~fortran", when="+hdf5~hdf5_compat+shared") + depends_on("hdf5~shared~cxx~mpi~fortran", when="+hdf5~hdf5_compat~shared") + + ############### + # Silo + ############### + # we are not using silo's fortran features + depends_on("silo~fortran", when="+silo+shared") + depends_on("silo~shared~fortran", when="+silo~shared") + + ############### + # ADIOS + ############### + depends_on("adios+mpi~hdf5+shared", when="+adios+mpi+shared") + depends_on("adios+mpi~hdf5~shared~blosc", when="+adios+mpi~shared") + depends_on("adios~mpi~hdf5+shared", when="+adios~mpi+shared") + depends_on("adios~mpi~hdf5~shared~blosc", when="+adios~mpi~shared") + + ####################### + # ZFP + ####################### + depends_on("zfp", when="+zfp") + + ####################### + # MPI + ####################### + depends_on("mpi", when="+mpi") + depends_on("py-mpi4py", when="+mpi+python") + + ####################### + # Documentation related + ####################### + depends_on("py-sphinx", when="+python+doc", type='build') + depends_on("py-sphinx-rtd-theme", when="+python+doc", type='build') + depends_on("doxygen", when="+doc+doxygen") + + # build phases used by this package + phases = ["configure", "build", "install"] + + def setup_environment(self, spack_env, run_env): + spack_env.set('CTEST_OUTPUT_ON_FAILURE', '1') + + def url_for_version(self, version): + """ + Provide proper url + """ + v = str(version) + if v == "0.2.0": + return "https://github.com/LLNL/conduit/archive/v0.2.0.tar.gz" + elif v == "0.2.1": + return "https://github.com/LLNL/conduit/archive/v0.2.1.tar.gz" + else: + # starting with v 0.3.0, conduit uses BLT + # (https://github.com/llnl/blt) as a submodule, since github does + # not automatically package source from submodules, conduit + # provides a custom src tarball + return "https://github.com/LLNL/conduit/releases/download/v{0}/conduit-v{1}-src-with-blt.tar.gz".format(v, v) + return url + + def configure(self, spec, prefix): + """ + Configure Conduit. + """ + with working_dir('spack-build', create=True): + py_site_pkgs_dir = None + if "+python" in spec: + py_site_pkgs_dir = site_packages_dir + + host_cfg_fname = self.create_host_config(spec, + prefix, + py_site_pkgs_dir) + # save this filename for + # other package recipe steps to access + self.host_cfg_fname = host_cfg_fname + cmake_args = [] + # if we have a static build, we need to avoid any of + # spack's default cmake settings related to rpaths + # (see: https://github.com/spack/spack/issues/2658) + if "+shared" in spec: + cmake_args.extend(std_cmake_args) + else: + for arg in std_cmake_args: + if arg.count("RPATH") == 0: + cmake_args.append(arg) + cmake_args.extend(["-C", host_cfg_fname, "../src"]) + print("Configuring Conduit...") + cmake(*cmake_args) + + def build(self, spec, prefix): + """ + Build Conduit. + """ + with working_dir('spack-build'): + print("Building Conduit...") + make() + + @run_after('build') + @on_package_attributes(run_tests=True) + def test(self): + with working_dir('spack-build'): + print("Running Conduit Unit Tests...") + make("test") + + def install(self, spec, prefix): + """ + Install Conduit. + """ + with working_dir('spack-build'): + make("install") + # install copy of host config for provenance + print("Installing Conduit CMake Host Config File...") + install(self.host_cfg_fname, prefix) + + @run_after('install') + @on_package_attributes(run_tests=True) + def check_install(self): + """ + Checks the spack install of conduit using conduit's + using-with-cmake example + """ + print("Checking Conduit installation...") + spec = self.spec + install_prefix = spec.prefix + example_src_dir = join_path(install_prefix, + "examples", + "conduit", + "using-with-cmake") + print("Checking using-with-cmake example...") + with working_dir("check-conduit-using-with-cmake-example", + create=True): + cmake_args = ["-DCONDUIT_DIR={0}".format(install_prefix), + example_src_dir] + cmake(*cmake_args) + make() + example = Executable('./conduit_example') + example() + print("Checking using-with-make example...") + example_src_dir = join_path(install_prefix, + "examples", + "conduit", + "using-with-make") + example_files = glob.glob(join_path(example_src_dir, "*")) + with working_dir("check-conduit-using-with-make-example", + create=True): + for example_file in example_files: + shutil.copy(example_file, ".") + make("CONDUIT_DIR={0}".format(install_prefix)) + example = Executable('./conduit_example') + example() + + def create_host_config(self, spec, prefix, py_site_pkgs_dir=None): + """ + This method creates a 'host-config' file that specifies + all of the options used to configure and build conduit. + + For more details about 'host-config' files see: + http://software.llnl.gov/conduit/building.html + + Note: + The `py_site_pkgs_dir` arg exists to allow a package that + subclasses this package provide a specific site packages + dir when calling this function. `py_site_pkgs_dir` should + be an absolute path or `None`. + + This is necessary because the spack `site_packages_dir` + var will not exist in the base class. For more details + on this issue see: https://github.com/spack/spack/issues/6261 + """ + + ####################### + # Compiler Info + ####################### + c_compiler = env["SPACK_CC"] + cpp_compiler = env["SPACK_CXX"] + f_compiler = None + + if self.compiler.fc: + # even if this is set, it may not exist so do one more sanity check + f_compiler = which(env["SPACK_FC"]) + + ####################################################################### + # By directly fetching the names of the actual compilers we appear + # to doing something evil here, but this is necessary to create a + # 'host config' file that works outside of the spack install env. + ####################################################################### + + sys_type = spec.architecture + # if on llnl systems, we can use the SYS_TYPE + if "SYS_TYPE" in env: + sys_type = env["SYS_TYPE"] + + ############################################## + # Find and record what CMake is used + ############################################## + + cmake_exe = spec['cmake'].command.path + host_cfg_fname = "%s-%s-%s-conduit.cmake" % (socket.gethostname(), + sys_type, + spec.compiler) + + cfg = open(host_cfg_fname, "w") + cfg.write("##################################\n") + cfg.write("# spack generated host-config\n") + cfg.write("##################################\n") + cfg.write("# {0}-{1}\n".format(sys_type, spec.compiler)) + cfg.write("##################################\n\n") + + # Include path to cmake for reference + cfg.write("# cmake from spack \n") + cfg.write("# cmake executable path: %s\n\n" % cmake_exe) + + ####################### + # Compiler Settings + ####################### + + cfg.write("#######\n") + cfg.write("# using %s compiler spec\n" % spec.compiler) + cfg.write("#######\n\n") + cfg.write("# c compiler used by spack\n") + cfg.write(cmake_cache_entry("CMAKE_C_COMPILER", c_compiler)) + cfg.write("# cpp compiler used by spack\n") + cfg.write(cmake_cache_entry("CMAKE_CXX_COMPILER", cpp_compiler)) + + cfg.write("# fortran compiler used by spack\n") + if "+fortran" in spec and f_compiler is not None: + cfg.write(cmake_cache_entry("ENABLE_FORTRAN", "ON")) + cfg.write(cmake_cache_entry("CMAKE_Fortran_COMPILER", + f_compiler.path)) + else: + cfg.write("# no fortran compiler found\n\n") + cfg.write(cmake_cache_entry("ENABLE_FORTRAN", "OFF")) + + if "+shared" in spec: + cfg.write(cmake_cache_entry("BUILD_SHARED_LIBS", "ON")) + else: + cfg.write(cmake_cache_entry("BUILD_SHARED_LIBS", "OFF")) + + # extra fun for blueos + if 'blueos_3' in sys_type and "+fortran" in spec: + if 'xl@coral' in os.getenv('SPACK_COMPILER_SPEC', ""): + # Fix missing std linker flag in xlc compiler + cfg.write(cmake_cache_entry("BLT_FORTRAN_FLAGS", + "-WF,-C! -qxlf2003=polymorphic")) + # Conduit can't link C++ into fortran for this spec, but works + # fine in host code + cfg.write(cmake_cache_entry("ENABLE_TESTS", "OFF")) + + ####################### + # Unit Tests + ####################### + if "+test" in spec: + cfg.write(cmake_cache_entry("ENABLE_TESTS", "ON")) + else: + cfg.write(cmake_cache_entry("ENABLE_TESTS", "OFF")) + + ####################### + # Python + ####################### + + cfg.write("# Python Support\n") + + if "+python" in spec and "+shared" in spec: + cfg.write("# Enable python module builds\n") + cfg.write(cmake_cache_entry("ENABLE_PYTHON", "ON")) + cfg.write("# python from spack \n") + cfg.write(cmake_cache_entry("PYTHON_EXECUTABLE", + spec['python'].command.path)) + # only set dest python site packages dir if passed + if py_site_pkgs_dir: + cfg.write(cmake_cache_entry("PYTHON_MODULE_INSTALL_PREFIX", + py_site_pkgs_dir)) + else: + cfg.write(cmake_cache_entry("ENABLE_PYTHON", "OFF")) + + if "+doc" in spec: + if "+python" in spec: + cfg.write(cmake_cache_entry("ENABLE_DOCS", "ON")) + + cfg.write("# sphinx from spack \n") + sphinx_build_exe = join_path(spec['py-sphinx'].prefix.bin, + "sphinx-build") + cfg.write(cmake_cache_entry("SPHINX_EXECUTABLE", + sphinx_build_exe)) + if "+doxygen" in spec: + cfg.write("# doxygen from uberenv\n") + doxygen_exe = spec['doxygen'].command.path + cfg.write(cmake_cache_entry("DOXYGEN_EXECUTABLE", doxygen_exe)) + else: + cfg.write(cmake_cache_entry("ENABLE_DOCS", "OFF")) + + ####################### + # MPI + ####################### + + cfg.write("# MPI Support\n") + + if "+mpi" in spec: + mpicc_path = spec['mpi'].mpicc + mpicxx_path = spec['mpi'].mpicxx + mpifc_path = spec['mpi'].mpifc + # if we are using compiler wrappers on cray systems + # use those for mpi wrappers, b/c spec['mpi'].mpicxx + # etc make return the spack compiler wrappers + # which can trip up mpi detection in CMake 3.14 + if cpp_compiler == "CC": + mpicc_path = "cc" + mpicxx_path = "CC" + mpifc_path = "ftn" + cfg.write(cmake_cache_entry("ENABLE_MPI", "ON")) + cfg.write(cmake_cache_entry("MPI_C_COMPILER", mpicc_path )) + cfg.write(cmake_cache_entry("MPI_CXX_COMPILER", mpicxx_path )) + cfg.write(cmake_cache_entry("MPI_Fortran_COMPILER", mpifc_path )) + mpiexe_bin = join_path(spec['mpi'].prefix.bin, 'mpiexec') + if os.path.isfile(mpiexe_bin): + # starting with cmake 3.10, FindMPI expects MPIEXEC_EXECUTABLE + # vs the older versions which expect MPIEXEC + if self.spec["cmake"].satisfies('@3.10:'): + cfg.write(cmake_cache_entry("MPIEXEC_EXECUTABLE", + mpiexe_bin)) + else: + cfg.write(cmake_cache_entry("MPIEXEC", + mpiexe_bin)) + else: + cfg.write(cmake_cache_entry("ENABLE_MPI", "OFF")) + + ####################### + # ZFP + ####################### + cfg.write("# zfp from spack \n") + if "+zfp" in spec: + cfg.write(cmake_cache_entry("ZFP_DIR", spec['zfp'].prefix)) + else: + cfg.write("# zfp not built by spack \n") + + ####################################################################### + # I/O Packages + ####################################################################### + + cfg.write("# I/O Packages\n\n") + + ####################### + # HDF5 + ####################### + + cfg.write("# hdf5 from spack \n") + + if "+hdf5" in spec: + cfg.write(cmake_cache_entry("HDF5_DIR", spec['hdf5'].prefix)) + # extra fun for BG/Q + if 'bgqos_0' in sys_type: + cfg.write(cmake_cache_entry('HDF5_C_LIBRARY_m', + '-lm', 'STRING')) + cfg.write(cmake_cache_entry('HDF5_C_LIBRARY_dl', + '-ldl', 'STRING')) + else: + cfg.write("# hdf5 not built by spack \n") + + ####################### + # Silo + ####################### + + cfg.write("# silo from spack \n") + + if "+silo" in spec: + cfg.write(cmake_cache_entry("SILO_DIR", spec['silo'].prefix)) + else: + cfg.write("# silo not built by spack \n") + + ####################### + # ADIOS + ####################### + + cfg.write("# ADIOS from spack \n") + + if "+adios" in spec: + cfg.write(cmake_cache_entry("ADIOS_DIR", spec['adios'].prefix)) + else: + cfg.write("# adios not built by spack \n") + + cfg.write("##################################\n") + cfg.write("# end spack generated host-config\n") + cfg.write("##################################\n") + cfg.close() + + host_cfg_fname = os.path.abspath(host_cfg_fname) + tty.info("spack generated conduit host-config file: " + host_cfg_fname) + return host_cfg_fname diff --git a/scripts/uberenv/project.json b/scripts/uberenv/project.json index 72e35ecc6..a3e8c62fb 100644 --- a/scripts/uberenv/project.json +++ b/scripts/uberenv/project.json @@ -5,5 +5,6 @@ "spack_branch": "task/2019_10_update_conduit", "spack_activate" : {"py-numpy" : ["+python"], "py-sphinx": ["+python","+doc"], + "py-mpi4py" : ["+python", "+mpi"], "py-sphinx-rtd-theme": ["+python","+doc"] } } diff --git a/src/cmake/SetupTests.cmake b/src/cmake/SetupTests.cmake index 391ad6a1a..ccf2b8588 100644 --- a/src/cmake/SetupTests.cmake +++ b/src/cmake/SetupTests.cmake @@ -151,12 +151,12 @@ function(add_python_test ) if(DEFINED ENV{PYTHONPATH}) set(py_path "$ENV{PYTHONPATH}${ENV_PATH_SEP}") endif() - set_property(TEST ${TEST} + set_property(TEST ${args_TEST} PROPERTY ENVIRONMENT "PYTHONPATH=${py_path}${CMAKE_BINARY_DIR}/python-modules/${ENV_PATH_SEP}${CMAKE_CURRENT_SOURCE_DIR}") if(WIN32) # proper path to dlls for vstudio and proper path for non config based gen (nmake, etc) - set_property(TEST ${TEST} + set_property(TEST ${args_TEST} APPEND PROPERTY ENVIRONMENT "PATH=${CMAKE_BINARY_DIR}/bin/${ENV_PATH_SEP}${CMAKE_BINARY_DIR}/bin/$/${ENV_PATH_SEP}$ENV{PATH}") @@ -169,6 +169,74 @@ function(add_python_test ) endfunction(add_python_test) +##------------------------------------------------------------------------------ +## - Builds and adds a test that uses python and mpi +## +## add_python_mpi_test( TEST test NUM_MPI_TASKS 2 ) +##------------------------------------------------------------------------------ +function(add_python_mpi_test) + + set(options) + set(singleValueArgs TEST NUM_MPI_TASKS FOLDER) + + # parse our arguments + cmake_parse_arguments(args + "${options}" + "${singleValueArgs}" + "${multiValueArgs}" ${ARGN} ) + + message(STATUS " [*] Adding Python-based MPI Unit Test: ${args_TEST}") + set(test_command ${PYTHON_EXECUTABLE} -B -m unittest -v ${args_TEST}) + + # Handle mpi + if ( ${args_NUM_MPI_TASKS} ) + set(test_command ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${args_NUM_MPI_TASKS} ${test_command} ) + endif() + + add_test(NAME ${args_TEST} + COMMAND ${test_command} ) + + # use proper env var path sep for current platform + if(WIN32) + set(ENV_PATH_SEP "\\;") + else() + set(ENV_PATH_SEP ":") + endif() + # make sure python can pick up the modules we built + # if python path is already set -- we need to append to it + # this is important for running in spack's build-env + set(py_path "") + if(DEFINED ENV{PYTHONPATH}) + set(py_path "$ENV{PYTHONPATH}${ENV_PATH_SEP}") + endif() + + set_property(TEST ${args_TEST} + PROPERTY + ENVIRONMENT "PYTHONPATH=${py_path}${CMAKE_BINARY_DIR}/python-modules/${ENV_PATH_SEP}${CMAKE_CURRENT_SOURCE_DIR}") + if(WIN32) + # proper path to dlls for vstudio and proper path for non config based gen (nmake, etc) + set_property(TEST ${args_TEST} + APPEND + PROPERTY + ENVIRONMENT "PATH=${CMAKE_BINARY_DIR}/bin/${ENV_PATH_SEP}${CMAKE_BINARY_DIR}/bin/$/${ENV_PATH_SEP}$ENV{PATH}") + endif() + + ########################################################################### + # Newer versions of OpenMPI require OMPI_MCA_rmaps_base_oversubscribe=1 + # to run with more tasks than actual cores + # Since this is an OpenMPI specific env var, it shouldn't interfere + # with other mpi implementations. + ########################################################################### + set_property(TEST ${args_TEST} + APPEND + PROPERTY ENVIRONMENT "OMPI_MCA_rmaps_base_oversubscribe=1") + + # set folder if passed + if( DEFINED args_FOLDER ) + blt_set_target_folder(TARGET ${args_TEST} FOLDER ${args_FOLDER}) + endif() + +endfunction() ##------------------------------------------------------------------------------ ## - Adds a fortran based unit test diff --git a/src/docs/sphinx/relay_mpi.rst b/src/docs/sphinx/relay_mpi.rst index 8cd99e58d..7903067ee 100644 --- a/src/docs/sphinx/relay_mpi.rst +++ b/src/docs/sphinx/relay_mpi.rst @@ -123,6 +123,153 @@ For both point to point and collectives, here is the basic logic for how input N +Python Relay MPI Module +------------------------ + +Relay MPI is supported in Python via the conduit.relay.mpi module. +Methods take Fortran-style MPI communicator handles which are effectively integers. +(We hope to also support direct use of `mpi4py` communicator objects in the future.) + +Use the following to get a handle from the `mpi4py` world communicator: + +.. code-block:: python + + from mpi4py import MPI + comm_id = MPI.COMM_WORLD.py2f() + + +Python Relay MPI Module Examples +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Send and Receive Using Schema +++++++++++++++++++++++++++++++ + +* **Python Source:** + +.. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_python_relay_mpi_examples.py + :start-after: BEGIN_EXAMPLE("py_mpi_send_and_recv_using_schema") + :end-before: END_EXAMPLE("py_mpi_send_and_recv_using_schema") + :language: python + :dedent: 8 + +* **Output:** + +.. literalinclude:: t_conduit_docs_python_tutorial_relay_mpi_examples_out.txt + :start-after: BEGIN_EXAMPLE("py_mpi_send_and_recv_using_schema") + :end-before: END_EXAMPLE("py_mpi_send_and_recv_using_schema") + :dedent: 4 + + +Send and Receive +++++++++++++++++++ + +* **Python Source:** + +.. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_python_relay_mpi_examples.py + :start-after: BEGIN_EXAMPLE("py_mpi_send_and_recv")) + :end-before: END_EXAMPLE("py_mpi_send_and_recv") + :language: python + :dedent: 8 + +* **Output:** + +.. literalinclude:: t_conduit_docs_python_tutorial_relay_mpi_examples_out.txt + :start-after: BEGIN_EXAMPLE("py_mpi_send_and_recv")) + :end-before: END_EXAMPLE("py_mpi_send_and_recv") + :dedent: 4 + +Send and Receive +++++++++++++++++++ + +* **Python Source:** + +.. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_python_relay_mpi_examples.py + :start-after: BEGIN_EXAMPLE("py_mpi_send_and_recv") + :end-before: END_EXAMPLE("py_mpi_send_and_recv") + :language: python + :dedent: 8 + +* **Output:** + +.. literalinclude:: t_conduit_docs_python_tutorial_relay_mpi_examples_out.txt + :start-after: BEGIN_EXAMPLE("py_mpi_send_and_recv") + :end-before: END_EXAMPLE("py_mpi_send_and_recv") + :dedent: 4 + +Sum All Reduce +++++++++++++++++++ + +* **Python Source:** + +.. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_python_relay_mpi_examples.py + :start-after: BEGIN_EXAMPLE("py_mpi_sum_all_reduce") + :end-before: END_EXAMPLE("py_mpi_sum_all_reduce") + :language: python + :dedent: 8 + +* **Output:** + +.. literalinclude:: t_conduit_docs_python_tutorial_relay_mpi_examples_out.txt + :start-after: BEGIN_EXAMPLE("py_mpi_sum_all_reduce") + :end-before: END_EXAMPLE("py_mpi_sum_all_reduce") + :dedent: 4 + +Broadcast Using Schema ++++++++++++++++++++++++ + +* **Python Source:** + +.. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_python_relay_mpi_examples.py + :start-after: BEGIN_EXAMPLE("py_mpi_bcast_using_schema") + :end-before: END_EXAMPLE("py_mpi_bcast_using_schema") + :language: python + :dedent: 8 + +* **Output:** + +.. literalinclude:: t_conduit_docs_python_tutorial_relay_mpi_examples_out.txt + :start-after: BEGIN_EXAMPLE("py_mpi_bcast_using_schema") + :end-before: END_EXAMPLE("py_mpi_bcast_using_schema") + :dedent: 4 + + +Broadcast ++++++++++++ + +* **Python Source:** + +.. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_python_relay_mpi_examples.py + :start-after: BEGIN_EXAMPLE("py_mpi_bcast")) + :end-before: END_EXAMPLE("py_mpi_bcast") + :language: python + :dedent: 8 + +* **Output:** + +.. literalinclude:: t_conduit_docs_python_tutorial_relay_mpi_examples_out.txt + :start-after: BEGIN_EXAMPLE("py_mpi_bcast")) + :end-before: END_EXAMPLE("py_mpi_bcast") + :dedent: 4 + + +All Gather Using Schema +++++++++++++++++++++++++ + +* **Python Source:** + +.. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_python_relay_mpi_examples.py + :start-after: BEGIN_EXAMPLE("py_mpi_all_gather_using_schema") + :end-before: END_EXAMPLE("py_mpi_all_gather_using_schema") + :language: python + :dedent: 8 + +* **Output:** + +.. literalinclude:: t_conduit_docs_python_tutorial_relay_mpi_examples_out.txt + :start-after: BEGIN_EXAMPLE("py_mpi_all_gather_using_schema") + :end-before: END_EXAMPLE("py_mpi_all_gather_using_schema") + :dedent: 4 + .. .. .. ================== ==================================== diff --git a/src/docs/sphinx/t_conduit_docs_python_tutorial_relay_mpi_examples_out.txt b/src/docs/sphinx/t_conduit_docs_python_tutorial_relay_mpi_examples_out.txt new file mode 100644 index 000000000..dcc34eed2 --- /dev/null +++ b/src/docs/sphinx/t_conduit_docs_python_tutorial_relay_mpi_examples_out.txt @@ -0,0 +1,116 @@ +UpdateCTestConfiguration from :/Users/harrison37/Work/github/llnl/conduit/build-debug/DartConfiguration.tcl +Parse Config file:/Users/harrison37/Work/github/llnl/conduit/build-debug/DartConfiguration.tcl +UpdateCTestConfiguration from :/Users/harrison37/Work/github/llnl/conduit/build-debug/DartConfiguration.tcl +Parse Config file:/Users/harrison37/Work/github/llnl/conduit/build-debug/DartConfiguration.tcl +Test project /Users/harrison37/Work/github/llnl/conduit/build-debug +Constructing a list of tests +Done constructing a list of tests +Updating test list for fixtures +Added 0 tests to meet fixture requirements +Checking test dependency graph... +Checking test dependency graph end +test 111 + Start 111: t_conduit_docs_tutorial_python_relay_mpi_examples + +111: Test command: /Users/harrison37/Work/github/llnl/conduit/uberenv_libs/spack/opt/spack/darwin-mojave-x86_64/clang-10.0.0-apple/openmpi-3.1.4-rnimf6bcbfwt6a6546uuylwg7brjkzxn/bin/mpiexec "-n" "2" "/Users/harrison37/Work/github/llnl/conduit/uberenv_libs/spack/opt/spack/darwin-mojave-x86_64/clang-10.0.0-apple/python-3.7.4-tuamu4qn6msg6bqa5hd7glbv5uxm5lup/bin/python3.7" "-B" "-m" "unittest" "-v" "t_conduit_docs_tutorial_python_relay_mpi_examples" +111: Environment variables: +111: PYTHONPATH=/Users/harrison37/Work/github/llnl/conduit/build-debug/python-modules/:/Users/harrison37/Work/github/llnl/conduit/src/tests/docs +111: OMPI_MCA_rmaps_base_oversubscribe=1 +111: Test timeout computed to be: 1500 +111: test_001_mpi_send_and_recv_using_schema (t_conduit_docs_tutorial_python_relay_mpi_examples.Conduit_Tutorial_Python_Relay_IO_Handle) ... test_001_mpi_send_and_recv_using_schema (t_conduit_docs_tutorial_python_relay_mpi_examples.Conduit_Tutorial_Python_Relay_IO_Handle) ... +111: BEGIN_EXAMPLE("py_mpi_send_and_recv_using_schema") +111: [rank: 0] sending: +111: a: +111: data: 1.0 +111: more_data: 2.0 +111: b: +111: my_string: "value" +111: +111: PyRelay_MPI_recv_using_schema +111: [rank: 1] received: +111: a: +111: data: 1.0 +111: more_data: 2.0 +111: b: +111: my_string: "value" +111: +111: +111: END_EXAMPLE("py_mpi_send_and_recv_using_schema") +111: ok +111: test_002_mpi_send_and_recv (t_conduit_docs_tutorial_python_relay_mpi_examples.Conduit_Tutorial_Python_Relay_IO_Handle) ... ok +111: test_002_mpi_send_and_recv (t_conduit_docs_tutorial_python_relay_mpi_examples.Conduit_Tutorial_Python_Relay_IO_Handle) ... +111: BEGIN_EXAMPLE("py_mpi_send_and_recv") +111: [rank: 0] sending: [0, 1, 4, 9] +111: +111: [rank: 1] received: [0, 1, 4, 9] +111: +111: +111: END_EXAMPLE("py_mpi_send_and_recv") +111: ok +111: test_003_mpi_bcast_using_schema (t_conduit_docs_tutorial_python_relay_mpi_examples.Conduit_Tutorial_Python_Relay_IO_Handle) ... ok +111: test_003_mpi_bcast_using_schema (t_conduit_docs_tutorial_python_relay_mpi_examples.Conduit_Tutorial_Python_Relay_IO_Handle) ... +111: BEGIN_EXAMPLE("py_mpi_bcast_using_schema") +111: [rank: 0] broadcasting: +111: a: +111: data: 1.0 +111: more_data: 2.0 +111: b: +111: my_string: "value" +111: +111: +111: END_EXAMPLE("py_mpi_bcast_using_schema") +111: [rank: 1] received: +111: a: +111: data: 1.0 +111: more_data: 2.0 +111: b: +111: my_string: "value" +111: +111: ok +111: test_004_mpi_bcast_using_schema (t_conduit_docs_tutorial_python_relay_mpi_examples.Conduit_Tutorial_Python_Relay_IO_Handle) ... ok +111: test_004_mpi_bcast_using_schema (t_conduit_docs_tutorial_python_relay_mpi_examples.Conduit_Tutorial_Python_Relay_IO_Handle) ... +111: BEGIN_EXAMPLE("py_mpi_bcast") +111: [rank: 0] broadcasting: [0, 1, 4, 9] +111: +111: [rank: 1] received: [0, 1, 4, 9] +111: +111: +111: END_EXAMPLE("py_mpi_bcast") +111: ok +111: test_005_mpi_sum_all_reduce (t_conduit_docs_tutorial_python_relay_mpi_examples.Conduit_Tutorial_Python_Relay_IO_Handle) ... ok +111: test_005_mpi_sum_all_reduce (t_conduit_docs_tutorial_python_relay_mpi_examples.Conduit_Tutorial_Python_Relay_IO_Handle) ... +111: BEGIN_EXAMPLE("py_mpi_sum_all_reduce") +111: [rank: 0] sum reduce result: [2, 2, 2, 2] +111: +111: +111: END_EXAMPLE("py_mpi_sum_all_reduce") +111: ok +111: test_006_mpi_all_gather (t_conduit_docs_tutorial_python_relay_mpi_examples.Conduit_Tutorial_Python_Relay_IO_Handle) ... ok +111: test_006_mpi_all_gather (t_conduit_docs_tutorial_python_relay_mpi_examples.Conduit_Tutorial_Python_Relay_IO_Handle) ... +111: BEGIN_EXAMPLE("py_mpi_all_gather_using_schema") +111: [rank: 0] all gather using schema result: +111: - [0, 0, 0, 0] +111: - [1, 1, 1, 1] +111: +111: +111: END_EXAMPLE("py_mpi_all_gather_using_schema") +111: ok +111: +111: ---------------------------------------------------------------------- +111: Ran 6 tests in 0.022s +111: +111: OK +111: ok +111: +111: ---------------------------------------------------------------------- +111: Ran 6 tests in 0.022s +111: +111: OK +1/1 Test #111: t_conduit_docs_tutorial_python_relay_mpi_examples ... Passed 0.28 sec + +The following tests passed: + t_conduit_docs_tutorial_python_relay_mpi_examples + +100% tests passed, 0 tests failed out of 1 + +Total Test time (real) = 0.31 sec diff --git a/src/libs/relay/python/CMakeLists.txt b/src/libs/relay/python/CMakeLists.txt index bf5d47a0d..2ea55be53 100644 --- a/src/libs/relay/python/CMakeLists.txt +++ b/src/libs/relay/python/CMakeLists.txt @@ -90,3 +90,25 @@ PYTHON_ADD_COMPILED_MODULE(conduit_relay_web_python # link with the proper libs (beyond python) target_link_libraries(conduit_relay_web_python conduit conduit_relay) + +################################################################ +# If we have mpi, add the conduit relay mpi module +################################################################ + +if(MPI_FOUND) + + # add relay mpi submodule + PYTHON_ADD_COMPILED_MODULE(conduit_relay_mpi_python + # dest + python-modules + # python module dir + conduit/relay/mpi + # c srcs + headers + conduit_relay_mpi_python.cpp + ${CMAKE_CURRENT_BINARY_DIR}/conduit_relay_python_exports.h) + + # link with the proper libs (beyond python) + target_link_libraries(conduit_relay_mpi_python conduit conduit_relay_mpi) + +endif() + diff --git a/src/libs/relay/python/conduit_relay_mpi_python.cpp b/src/libs/relay/python/conduit_relay_mpi_python.cpp new file mode 100644 index 000000000..2a3447f8b --- /dev/null +++ b/src/libs/relay/python/conduit_relay_mpi_python.cpp @@ -0,0 +1,2137 @@ +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// +// Copyright (c) 2014-2019, Lawrence Livermore National Security, LLC. +// +// Produced at the Lawrence Livermore National Laboratory +// +// LLNL-CODE-666778 +// +// All rights reserved. +// +// This file is part of Conduit. +// +// For details, see: http://software.llnl.gov/conduit/. +// +// Please also read conduit/LICENSE +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the disclaimer below. +// +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the disclaimer (as noted below) in the +// documentation and/or other materials provided with the distribution. +// +// * Neither the name of the LLNS/LLNL nor the names of its contributors may +// be used to endorse or promote products derived from this software without +// specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL SECURITY, +// LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +// OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING +// IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. +// +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// + + +//----------------------------------------------------------------------------- +// -- Python includes (these must be included first) -- +//----------------------------------------------------------------------------- +#include +#include +#include "bytesobject.h" + +//----------------------------------------------------------------------------- +// -- standard lib includes -- +//----------------------------------------------------------------------------- +#include +#include + +//---------------------------------------------------------------------------// +// conduit includes +//---------------------------------------------------------------------------// +#include "conduit.hpp" +#include "conduit_relay_mpi.hpp" + +#include "conduit_relay_python_exports.h" + +// conduit python module capi header +#include "conduit_python.hpp" + + +using namespace conduit; +using namespace conduit::relay::mpi; + +#if PY_MAJOR_VERSION >= 3 +#define IS_PY3K +#endif + +// use proper strdup +#ifdef CONDUIT_PLATFORM_WINDOWS + #define _conduit_strdup _strdup +#else + #define _conduit_strdup strdup +#endif + +//----------------------------------------------------------------------------- +// PyVarObject_TAIL is used at the end of each PyVarObject def +// to make sure we have the correct number of initializers across python +// versions. +//----------------------------------------------------------------------------- +#ifdef Py_TPFLAGS_HAVE_FINALIZE +#define PyVarObject_TAIL ,0 +#else +#define PyVarObject_TAIL +#endif + + +//----------------------------------------------------------------------------- +//----------------------------------------------------------------------------- +// Begin Functions to help with Python 2/3 Compatibility. +//----------------------------------------------------------------------------- +//----------------------------------------------------------------------------- + +#if defined(IS_PY3K) + +//----------------------------------------------------------------------------- +static int +PyString_Check(PyObject *o) +{ + return PyUnicode_Check(o); +} + +//----------------------------------------------------------------------------- +static char * +PyString_AsString(PyObject *py_obj) +{ + char *res = NULL; + if(PyUnicode_Check(py_obj)) + { + PyObject * temp_bytes = PyUnicode_AsEncodedString(py_obj, + "ASCII", + "strict"); // Owned reference + if(temp_bytes != NULL) + { + res = _conduit_strdup(PyBytes_AS_STRING(temp_bytes)); + Py_DECREF(temp_bytes); + } + else + { + // TODO: Error + } + } + else if(PyBytes_Check(py_obj)) + { + res = _conduit_strdup(PyBytes_AS_STRING(py_obj)); + } + else + { + // TODO: ERROR or auto convert? + } + + return res; +} + +//----------------------------------------------------------------------------- +static PyObject * +PyString_FromString(const char *s) +{ + return PyUnicode_FromString(s); +} + +//----------------------------------------------------------------------------- +static void +PyString_AsString_Cleanup(char *bytes) +{ + free(bytes); +} + + +//----------------------------------------------------------------------------- +static int +PyInt_Check(PyObject *o) +{ + return PyLong_Check(o); +} + +//----------------------------------------------------------------------------- +static long +PyInt_AsLong(PyObject *o) +{ + return PyLong_AsLong(o); +} + +#else // python 2.6+ + +//----------------------------------------------------------------------------- +#define PyString_AsString_Cleanup(c) { /* noop */ } + +#endif + +//----------------------------------------------------------------------------- +//----------------------------------------------------------------------------- +// End Functions to help with Python 2/3 Compatibility. +//----------------------------------------------------------------------------- +//----------------------------------------------------------------------------- + +//---------------------------------------------------------------------------// +struct PyRelay_MPI_Request +{ + PyObject_HEAD + Request request; +}; + +//---------------------------------------------------------------------------// +//---------------------------------------------------------------------------// +// +// relay::mpi::Request Object +// +//---------------------------------------------------------------------------// +//---------------------------------------------------------------------------// + +//---------------------------------------------------------------------------// +static PyObject * +PyRelay_MPI_Request_new(PyTypeObject *type, + PyObject*, // args -- unused + PyObject*) // kwds -- unused +{ + PyRelay_MPI_Request *self = (PyRelay_MPI_Request*)type->tp_alloc(type, 0); + + return ((PyObject*)self); +} + +//---------------------------------------------------------------------------// +static void +PyRelay_MPI_Request_dealloc(PyRelay_MPI_Request *self) +{ + Py_TYPE(self)->tp_free((PyObject*)self); +} + +//---------------------------------------------------------------------------// +static int +PyRelay_MPI_Request_init(PyRelay_MPI_Request * /*self*/) // self unused +{ + return 0; +} + +//---------------------------------------------------------------------------// +static Request * +PyRelay_MPI_Request_request_pointer(PyRelay_MPI_Request *py_req) +{ + return &py_req->request; +} + +//----------------------------------------------------------------------------// +// Request methods table +//----------------------------------------------------------------------------// +static PyMethodDef PyRelay_MPI_Request_METHODS[] = { + //-----------------------------------------------------------------------// + //-----------------------------------------------------------------------// + // end Request methods table + //-----------------------------------------------------------------------// + {NULL, NULL, 0, NULL} +}; + +//---------------------------------------------------------------------------// +static PyTypeObject PyRelay_MPI_Request_TYPE = { + PyVarObject_HEAD_INIT(NULL, 0) + "Request", + sizeof(PyRelay_MPI_Request), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)PyRelay_MPI_Request_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* as_mapping */ + 0, /* hash */ + 0, /* call */ + 0, /* str */ + 0, /* getattro */ + 0, /* setattro */ + 0, /* asbuffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* flags */ + "Conduit Relay MPI Request objects", + 0, /* traverse */ + 0, /* clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* iter */ + 0, /* iternext */ + PyRelay_MPI_Request_METHODS, /* METHODS */ + 0, /* MEMBERS */ + 0, /* get/set */ + 0, /* tp_base */ + 0, /* dict */ + 0, /* descr_get */ + 0, /* gescr_set */ + 0, /* dictoffset */ + (initproc)PyRelay_MPI_Request_init, + 0, /* alloc */ + PyRelay_MPI_Request_new, /* new */ + 0, /* tp_free */ + 0, /* tp_is_gc */ + 0, /* tp_bases */ + 0, /* tp_mro */ + 0, /* tp_cache */ + 0, /* tp_subclasses */ + 0, /* tp_weaklist */ + 0, + 0 + PyVarObject_TAIL +}; + + +//---------------------------------------------------------------------------// +static int +PyRelay_MPI_Request_Check(PyObject* obj) +{ + return (PyObject_TypeCheck(obj, &PyRelay_MPI_Request_TYPE)); +} + + +//----------------------------------------------------------------------------- +/// MPI Module Functions +//----------------------------------------------------------------------------- + +//---------------------------------------------------------------------------// +// conduit::relay::mpi::about +//---------------------------------------------------------------------------// +static PyObject * +PyRelay_MPI_about() +{ + //create and return a node with the result of about + PyObject *py_node_res = PyConduit_Node_Python_Create(); + Node *node = PyConduit_Node_Get_Node_Ptr(py_node_res); + conduit::relay::mpi::about(*node); + return (PyObject*)py_node_res; +} + +//---------------------------------------------------------------------------// +// conduit::relay::mpi::rank +//---------------------------------------------------------------------------// +static PyObject * +PyRelay_MPI_rank(PyObject *, //self + PyObject *args, + PyObject *kwargs) +{ + Py_ssize_t mpi_comm_id; + + // TODO: future also accept mpi4py comm + + static const char *kwlist[] = {"comm", + NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, + kwargs, + "n", + const_cast(kwlist), + &mpi_comm_id)) + { + return (NULL); + } + + // get c mpi comm hnd + MPI_Comm comm = MPI_Comm_f2c(mpi_comm_id); + + int rank = -1; + + try + { + rank = relay::mpi::rank(comm); + } + catch(conduit::Error e) + { + PyErr_SetString(PyExc_Exception, + e.message().c_str()); + return NULL; + } + + return PyLong_FromLong((long)rank); +} + +//---------------------------------------------------------------------------// +// conduit::relay::mpi::size +//---------------------------------------------------------------------------// +static PyObject * +PyRelay_MPI_size(PyObject *, //self + PyObject *args, + PyObject *kwargs) +{ + Py_ssize_t mpi_comm_id; + + // TODO: future also accept mpi4py comm + + static const char *kwlist[] = {"comm", + NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, + kwargs, + "n", + const_cast(kwlist), + &mpi_comm_id)) + { + return (NULL); + } + + // get c mpi comm hnd + MPI_Comm comm = MPI_Comm_f2c(mpi_comm_id); + + int size = 0; + + try + { + size = relay::mpi::size(comm); + } + catch(conduit::Error e) + { + PyErr_SetString(PyExc_Exception, + e.message().c_str()); + return NULL; + } + + return PyLong_FromLong((long)size); +} + +//----------------------------------------------------------------------------- +/// Standard MPI Send Recv +//----------------------------------------------------------------------------- + +//---------------------------------------------------------------------------// +// conduit::relay::mpi::send +//---------------------------------------------------------------------------// +static PyObject * +PyRelay_MPI_send(PyObject *, //self + PyObject *args, + PyObject *kwargs) +{ + PyObject *py_node = NULL; + Py_ssize_t dest; + Py_ssize_t tag; + Py_ssize_t mpi_comm_id; + + // TODO: future also accept mpi4py comm + + static const char *kwlist[] = {"node", + "dest", + "tag", + "comm", + NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, + kwargs, + "Onnn", + const_cast(kwlist), + &py_node, + &dest, + &tag, + &mpi_comm_id)) + { + return (NULL); + } + + if(!PyConduit_Node_Check(py_node)) + { + PyErr_SetString(PyExc_TypeError, + "'node' argument must be a " + "conduit.Node instance"); + return NULL; + } + + Node &node = *PyConduit_Node_Get_Node_Ptr(py_node); + + // get c mpi comm hnd + MPI_Comm comm = MPI_Comm_f2c(mpi_comm_id); + + try + { + relay::mpi::send(node, dest, tag, comm); + } + catch(conduit::Error e) + { + PyErr_SetString(PyExc_Exception, + e.message().c_str()); + return NULL; + } + + Py_RETURN_NONE; +} + +//---------------------------------------------------------------------------// +// conduit::relay::mpi::recv +//---------------------------------------------------------------------------// +static PyObject * +PyRelay_MPI_recv(PyObject *, //self + PyObject *args, + PyObject *kwargs) +{ + PyObject *py_node = NULL; + Py_ssize_t source; + Py_ssize_t tag; + Py_ssize_t mpi_comm_id; + + // TODO: future also accept mpi4py comm + + static const char *kwlist[] = {"node", + "source", + "tag", + "comm", + NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, + kwargs, + "Onnn", + const_cast(kwlist), + &py_node, + &source, + &tag, + &mpi_comm_id)) + { + return (NULL); + } + + if(!PyConduit_Node_Check(py_node)) + { + PyErr_SetString(PyExc_TypeError, + "'node' argument must be a " + "conduit.Node instance"); + return NULL; + } + + + Node &node = *PyConduit_Node_Get_Node_Ptr(py_node); + + // get c mpi comm hnd + MPI_Comm comm = MPI_Comm_f2c(mpi_comm_id); + + try + { + relay::mpi::recv(node, source, tag, comm); + } + catch(conduit::Error e) + { + PyErr_SetString(PyExc_Exception, + e.message().c_str()); + return NULL; + } + + Py_RETURN_NONE; +} + +//---------------------------------------------------------------------------// +// conduit::relay::mpi::send_using_schema +//---------------------------------------------------------------------------// +static PyObject * +PyRelay_MPI_send_using_schema(PyObject *, //self + PyObject *args, + PyObject *kwargs) +{ + PyObject *py_node = NULL; + Py_ssize_t dest; + Py_ssize_t tag; + Py_ssize_t mpi_comm_id; + + // TODO: future also accept mpi4py comm + + static const char *kwlist[] = {"node", + "dest", + "tag", + "comm", + NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, + kwargs, + "Onnn", + const_cast(kwlist), + &py_node, + &dest, + &tag, + &mpi_comm_id)) + { + return (NULL); + } + + if(!PyConduit_Node_Check(py_node)) + { + PyErr_SetString(PyExc_TypeError, + "'node' argument must be a " + "conduit.Node instance"); + return NULL; + } + + Node &node = *PyConduit_Node_Get_Node_Ptr(py_node); + + // get c mpi comm hnd + MPI_Comm comm = MPI_Comm_f2c(mpi_comm_id); + + try + { + relay::mpi::send_using_schema(node, dest, tag, comm); + } + catch(conduit::Error e) + { + PyErr_SetString(PyExc_Exception, + e.message().c_str()); + return NULL; + } + + Py_RETURN_NONE; +} + +//---------------------------------------------------------------------------// +// conduit::relay::mpi::recv_using_schema +//---------------------------------------------------------------------------// +static PyObject * +PyRelay_MPI_recv_using_schema(PyObject *, //self + PyObject *args, + PyObject *kwargs) +{ + PyObject *py_node = NULL; + Py_ssize_t source; + Py_ssize_t tag; + Py_ssize_t mpi_comm_id; + + // TODO: future also accept mpi4py comm + + static const char *kwlist[] = {"node", + "source", + "tag", + "comm", + NULL}; + +std::cout << "PyRelay_MPI_recv_using_schema" << std::endl; + + if (!PyArg_ParseTupleAndKeywords(args, + kwargs, + "Onnn", + const_cast(kwlist), + &py_node, + &source, + &tag, + &mpi_comm_id)) + { + std::cout << "PyRelay_MPI_recv_using_schema ansdsadiosnisaniods" << std::endl; + return (NULL); + } + + if(!PyConduit_Node_Check(py_node)) + { + PyErr_SetString(PyExc_TypeError, + "'node' argument must be a " + "conduit.Node instance"); + return NULL; + } + + Node &node = *PyConduit_Node_Get_Node_Ptr(py_node); + + // get c mpi comm hnd + MPI_Comm comm = MPI_Comm_f2c(mpi_comm_id); + + try + { + relay::mpi::recv_using_schema(node, source, tag, comm); + } + catch(conduit::Error e) + { + PyErr_SetString(PyExc_Exception, + e.message().c_str()); + return NULL; + } + + Py_RETURN_NONE; +} + +//----------------------------------------------------------------------------- +// TODO: Generic MPI Reduce +//----------------------------------------------------------------------------- +// //----------------------------------------------------------------------------- +// /// MPI Reduce +// //----------------------------------------------------------------------------- +// +// /// MPI reduce and all reduce methods. +// +// /// While the input does not need to be compact, +// /// reductions require all MPI ranks have identical compact representations. +// +// /// These methods do not check across ranks for identical compact +// /// representation. +// +// /// Conduit empty, object, and list dtypes can not be reduced. +// +// /// If the send_node is not compact, it will be compacted prior to sending. +// +// /// for reduce on the root rank and all_reduce for all ranks: +// /// if the recv_node is compatible but not compact, data will be placed +// /// into a compact buffer, then read back out into the recv_node node. +// /// +// /// if the recv_node is not compatible, it will be reset to +// /// a compact compatible type. +// +// int CONDUIT_RELAY_API reduce(const Node &send_node, +// Node &recv_node, +// MPI_Op mpi_op, +// int root, +// MPI_Comm comm); +// +// int CONDUIT_RELAY_API all_reduce(const Node &send_node, +// Node &recv_node, +// MPI_Op mpi_op, +// MPI_Comm comm); +// + + + +//----------------------------------------------------------------------------- +/// MPI Reduce Helpers +//----------------------------------------------------------------------------- + +//---------------------------------------------------------------------------// +// conduit::relay::mpi::sum_reduce +//---------------------------------------------------------------------------// +static PyObject * +PyRelay_MPI_sum_reduce(PyObject *, //self + PyObject *args, + PyObject *kwargs) +{ + PyObject *py_src_node = NULL; + PyObject *py_recv_node = NULL; + Py_ssize_t root; + Py_ssize_t mpi_comm_id; + + // TODO: future also accept mpi4py comm + // TODO: We could use shared arg parsing for reduce helpers + // I started down that path, but opted to keep sep parsing + // to get over the finish line + + static const char *kwlist[] = {"send_node", + "recv_node", + "root", + "comm", + NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, + kwargs, + "OOnn", + const_cast(kwlist), + &py_src_node, + &py_recv_node, + &root, + &mpi_comm_id)) + { + return (NULL); + } + + if(!PyConduit_Node_Check(py_src_node)) + { + PyErr_SetString(PyExc_TypeError, + "'send_node' argument must be a " + "conduit.Node instance"); + return NULL; + } + + if(!PyConduit_Node_Check(py_recv_node)) + { + PyErr_SetString(PyExc_TypeError, + "'recv_node' argument must be a " + "conduit.Node instance"); + return NULL; + } + + Node &send_node = *PyConduit_Node_Get_Node_Ptr(py_src_node); + Node &recv_node = *PyConduit_Node_Get_Node_Ptr(py_recv_node); + + // get c mpi comm hnd + MPI_Comm comm = MPI_Comm_f2c(mpi_comm_id); + + try + { + relay::mpi::sum_reduce(send_node, + recv_node, + root, + comm); + } + catch(conduit::Error e) + { + PyErr_SetString(PyExc_Exception, + e.message().c_str()); + return NULL; + } + + Py_RETURN_NONE; +} + +//---------------------------------------------------------------------------// +// conduit::relay::mpi::min_reduce +//---------------------------------------------------------------------------// +static PyObject * +PyRelay_MPI_min_reduce(PyObject *, //self + PyObject *args, + PyObject *kwargs) +{ + PyObject *py_src_node = NULL; + PyObject *py_recv_node = NULL; + Py_ssize_t root; + Py_ssize_t mpi_comm_id; + + // TODO: future also accept mpi4py comm + // TODO: We could use shared arg parsing for reduce helpers + // I started down that path, but opted to keep sep parsing + // to get over the finish line + + static const char *kwlist[] = {"send_node", + "recv_node", + "root", + "comm", + NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, + kwargs, + "OOnn", + const_cast(kwlist), + &py_src_node, + &py_recv_node, + &root, + &mpi_comm_id)) + { + return (NULL); + } + + if(!PyConduit_Node_Check(py_src_node)) + { + PyErr_SetString(PyExc_TypeError, + "'send_node' argument must be a " + "conduit.Node instance"); + return NULL; + } + + if(!PyConduit_Node_Check(py_recv_node)) + { + PyErr_SetString(PyExc_TypeError, + "'recv_node' argument must be a " + "conduit.Node instance"); + return NULL; + } + + Node &send_node = *PyConduit_Node_Get_Node_Ptr(py_src_node); + Node &recv_node = *PyConduit_Node_Get_Node_Ptr(py_recv_node); + + // get c mpi comm hnd + MPI_Comm comm = MPI_Comm_f2c(mpi_comm_id); + + try + { + relay::mpi::min_reduce(send_node, + recv_node, + root, + comm); + } + catch(conduit::Error e) + { + PyErr_SetString(PyExc_Exception, + e.message().c_str()); + return NULL; + } + + Py_RETURN_NONE; +} + + +//---------------------------------------------------------------------------// +// conduit::relay::mpi::max_reduce +//---------------------------------------------------------------------------// +static PyObject * +PyRelay_MPI_max_reduce(PyObject *, //self + PyObject *args, + PyObject *kwargs) +{ + PyObject *py_src_node = NULL; + PyObject *py_recv_node = NULL; + Py_ssize_t root; + Py_ssize_t mpi_comm_id; + + // TODO: future also accept mpi4py comm + // TODO: We could use shared arg parsing for reduce helpers + // I started down that path, but opted to keep sep parsing + // to get over the finish line + + static const char *kwlist[] = {"send_node", + "recv_node", + "root", + "comm", + NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, + kwargs, + "OOnn", + const_cast(kwlist), + &py_src_node, + &py_recv_node, + &root, + &mpi_comm_id)) + { + return (NULL); + } + + if(!PyConduit_Node_Check(py_src_node)) + { + PyErr_SetString(PyExc_TypeError, + "'send_node' argument must be a " + "conduit.Node instance"); + return NULL; + } + + if(!PyConduit_Node_Check(py_recv_node)) + { + PyErr_SetString(PyExc_TypeError, + "'recv_node' argument must be a " + "conduit.Node instance"); + return NULL; + } + + Node &send_node = *PyConduit_Node_Get_Node_Ptr(py_src_node); + Node &recv_node = *PyConduit_Node_Get_Node_Ptr(py_recv_node); + + // get c mpi comm hnd + MPI_Comm comm = MPI_Comm_f2c(mpi_comm_id); + + try + { + relay::mpi::max_reduce(send_node, + recv_node, + root, + comm); + } + catch(conduit::Error e) + { + PyErr_SetString(PyExc_Exception, + e.message().c_str()); + return NULL; + } + + Py_RETURN_NONE; +} + + +//---------------------------------------------------------------------------// +// conduit::relay::mpi::prod_reduce +//---------------------------------------------------------------------------// +static PyObject * +PyRelay_MPI_prod_reduce(PyObject *, //self + PyObject *args, + PyObject *kwargs) +{ + PyObject *py_src_node = NULL; + PyObject *py_recv_node = NULL; + Py_ssize_t root; + Py_ssize_t mpi_comm_id; + + // TODO: future also accept mpi4py comm + // TODO: We could use shared arg parsing for reduce helpers + // I started down that path, but opted to keep sep parsing + // to get over the finish line + + static const char *kwlist[] = {"send_node", + "recv_node", + "root", + "comm", + NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, + kwargs, + "OOnn", + const_cast(kwlist), + &py_src_node, + &py_recv_node, + &root, + &mpi_comm_id)) + { + return (NULL); + } + + if(!PyConduit_Node_Check(py_src_node)) + { + PyErr_SetString(PyExc_TypeError, + "'send_node' argument must be a " + "conduit.Node instance"); + return NULL; + } + + if(!PyConduit_Node_Check(py_recv_node)) + { + PyErr_SetString(PyExc_TypeError, + "'recv_node' argument must be a " + "conduit.Node instance"); + return NULL; + } + + Node &send_node = *PyConduit_Node_Get_Node_Ptr(py_src_node); + Node &recv_node = *PyConduit_Node_Get_Node_Ptr(py_recv_node); + + // get c mpi comm hnd + MPI_Comm comm = MPI_Comm_f2c(mpi_comm_id); + + try + { + relay::mpi::prod_reduce(send_node, + recv_node, + root, + comm); + } + catch(conduit::Error e) + { + PyErr_SetString(PyExc_Exception, + e.message().c_str()); + return NULL; + } + + Py_RETURN_NONE; +} + +//---------------------------------------------------------------------------// +// conduit::relay::mpi::sum_all_reduce +//---------------------------------------------------------------------------// +static PyObject * +PyRelay_MPI_sum_all_reduce(PyObject *, //self + PyObject *args, + PyObject *kwargs) +{ + PyObject *py_src_node = NULL; + PyObject *py_recv_node = NULL; + Py_ssize_t mpi_comm_id; + + // TODO: future also accept mpi4py comm + // TODO: We could use shared arg parsing for reduce helpers + // I started down that path, but opted to keep sep parsing + // to get over the finish line + + static const char *kwlist[] = {"send_node", + "recv_node", + "comm", + NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, + kwargs, + "OOn", + const_cast(kwlist), + &py_src_node, + &py_recv_node, + &mpi_comm_id)) + { + return (NULL); + } + + if(!PyConduit_Node_Check(py_src_node)) + { + PyErr_SetString(PyExc_TypeError, + "'send_node' argument must be a " + "conduit.Node instance"); + return NULL; + } + + if(!PyConduit_Node_Check(py_recv_node)) + { + PyErr_SetString(PyExc_TypeError, + "'recv_node' argument must be a " + "conduit.Node instance"); + return NULL; + } + + Node &send_node = *PyConduit_Node_Get_Node_Ptr(py_src_node); + Node &recv_node = *PyConduit_Node_Get_Node_Ptr(py_recv_node); + + // get c mpi comm hnd + MPI_Comm comm = MPI_Comm_f2c(mpi_comm_id); + + try + { + relay::mpi::sum_all_reduce(send_node, + recv_node, + comm); + } + catch(conduit::Error e) + { + PyErr_SetString(PyExc_Exception, + e.message().c_str()); + return NULL; + } + + Py_RETURN_NONE; +} + +//---------------------------------------------------------------------------// +// conduit::relay::mpi::min_all_reduce +//---------------------------------------------------------------------------// +static PyObject * +PyRelay_MPI_min_all_reduce(PyObject *, //self + PyObject *args, + PyObject *kwargs) +{ + PyObject *py_src_node = NULL; + PyObject *py_recv_node = NULL; + Py_ssize_t mpi_comm_id; + + + // TODO: future also accept mpi4py comm + // TODO: We could use shared arg parsing for reduce helpers + // I started down that path, but opted to keep sep parsing + // to get over the finish line + + static const char *kwlist[] = {"send_node", + "recv_node", + "comm", + NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, + kwargs, + "OOn", + const_cast(kwlist), + &py_src_node, + &py_recv_node, + &mpi_comm_id)) + { + return (NULL); + } + + if(!PyConduit_Node_Check(py_src_node)) + { + PyErr_SetString(PyExc_TypeError, + "'send_node' argument must be a " + "conduit.Node instance"); + return NULL; + } + + if(!PyConduit_Node_Check(py_recv_node)) + { + PyErr_SetString(PyExc_TypeError, + "'recv_node' argument must be a " + "conduit.Node instance"); + return NULL; + } + + Node &send_node = *PyConduit_Node_Get_Node_Ptr(py_src_node); + Node &recv_node = *PyConduit_Node_Get_Node_Ptr(py_recv_node); + + // get c mpi comm hnd + MPI_Comm comm = MPI_Comm_f2c(mpi_comm_id); + + try + { + relay::mpi::min_all_reduce(send_node, + recv_node, + comm); + } + catch(conduit::Error e) + { + PyErr_SetString(PyExc_Exception, + e.message().c_str()); + return NULL; + } + + Py_RETURN_NONE; +} + +//---------------------------------------------------------------------------// +// conduit::relay::mpi::max_all_reduce +//---------------------------------------------------------------------------// +static PyObject * +PyRelay_MPI_max_all_reduce(PyObject *, //self + PyObject *args, + PyObject *kwargs) +{ + PyObject *py_src_node = NULL; + PyObject *py_recv_node = NULL; + Py_ssize_t mpi_comm_id; + + // TODO: future also accept mpi4py comm + // TODO: We could use shared arg parsing for reduce helpers + // I started down that path, but opted to keep sep parsing + // to get over the finish line + + static const char *kwlist[] = {"send_node", + "recv_node", + "comm", + NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, + kwargs, + "OOn", + const_cast(kwlist), + &py_src_node, + &py_recv_node, + &mpi_comm_id)) + { + return (NULL); + } + + if(!PyConduit_Node_Check(py_src_node)) + { + PyErr_SetString(PyExc_TypeError, + "'send_node' argument must be a " + "conduit.Node instance"); + return NULL; + } + + if(!PyConduit_Node_Check(py_recv_node)) + { + PyErr_SetString(PyExc_TypeError, + "'recv_node' argument must be a " + "conduit.Node instance"); + return NULL; + } + + Node &send_node = *PyConduit_Node_Get_Node_Ptr(py_src_node); + Node &recv_node = *PyConduit_Node_Get_Node_Ptr(py_recv_node); + + // get c mpi comm hnd + MPI_Comm comm = MPI_Comm_f2c(mpi_comm_id); + + try + { + relay::mpi::max_all_reduce(send_node, + recv_node, + comm); + } + catch(conduit::Error e) + { + PyErr_SetString(PyExc_Exception, + e.message().c_str()); + return NULL; + } + + Py_RETURN_NONE; +} + +//---------------------------------------------------------------------------// +// conduit::relay::mpi::prod_all_reduce +//---------------------------------------------------------------------------// +static PyObject * +PyRelay_MPI_prod_all_reduce(PyObject *, //self + PyObject *args, + PyObject *kwargs) +{ + PyObject *py_src_node = NULL; + PyObject *py_recv_node = NULL; + Py_ssize_t mpi_comm_id; + + // TODO: future also accept mpi4py comm + // TODO: We could use shared arg parsing for reduce helpers + // I started down that path, but opted to keep sep parsing + // to get over the finish line + + static const char *kwlist[] = {"send_node", + "recv_node", + "comm", + NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, + kwargs, + "OOn", + const_cast(kwlist), + &py_src_node, + &py_recv_node, + &mpi_comm_id)) + { + return (NULL); + } + + if(!PyConduit_Node_Check(py_src_node)) + { + PyErr_SetString(PyExc_TypeError, + "'send_node' argument must be a " + "conduit.Node instance"); + return NULL; + } + + if(!PyConduit_Node_Check(py_recv_node)) + { + PyErr_SetString(PyExc_TypeError, + "'recv_node' argument must be a " + "conduit.Node instance"); + return NULL; + } + + Node &send_node = *PyConduit_Node_Get_Node_Ptr(py_src_node); + Node &recv_node = *PyConduit_Node_Get_Node_Ptr(py_recv_node); + + // get c mpi comm hnd + MPI_Comm comm = MPI_Comm_f2c(mpi_comm_id); + + try + { + relay::mpi::prod_all_reduce(send_node, + recv_node, + comm); + } + catch(conduit::Error e) + { + PyErr_SetString(PyExc_Exception, + e.message().c_str()); + return NULL; + } + + Py_RETURN_NONE; +} + +//----------------------------------------------------------------------------- +// TODO: Async MPI Send Recv +//----------------------------------------------------------------------------- +// + +//----------------------------------------------------------------------------- +/// Async MPI Send Recv +//----------------------------------------------------------------------------- + + +//---------------------------------------------------------------------------// +// conduit::relay::mpi::isend +//---------------------------------------------------------------------------// +// static PyObject * +// PyRelay_MPI_isend(PyObject *, //self +// PyObject *args, +// PyObject *kwargs) +// { +// PyObject *py_node = NULL; +// Py_ssize_t dest; +// Py_ssize_t tag; +// Py_ssize_t mpi_comm_id; +// PyObject *py_mpi_request = NULL; +// +// // TODO: future also accept mpi4py comm +// +// static const char *kwlist[] = {"node", +// "dest", +// "tag", +// "comm", +// "request", +// NULL}; +// +// if (!PyArg_ParseTupleAndKeywords(args, +// kwargs, +// "OnnnO", +// const_cast(kwlist), +// &py_node, +// &dest, +// &tag, +// &mpi_comm_id, +// &py_mpi_request)) +// { +// return (NULL); +// } +// +// if(!PyConduit_Node_Check(py_node)) +// { +// PyErr_SetString(PyExc_TypeError, +// "'node' argument must be a " +// "conduit.Node instance"); +// return NULL; +// } +// +// if(!PyRelay_MPI_Request_Check(py_mpi_request)) +// { +// PyErr_SetString(PyExc_TypeError, +// "'request' argument must be a " +// "conduit.relay.mpi Request instance"); +// return NULL; +// } +// +// Node &node = *PyConduit_Node_Get_Node_Ptr(py_node); +// Request *request = PyRelay_MPI_Request_request_pointer((PyRelay_MPI_Request*)py_mpi_request); +// +// // get c mpi comm hnd +// MPI_Comm comm = MPI_Comm_f2c(mpi_comm_id); +// +// try +// { +// relay::mpi::isend(node, +// dest, +// tag, +// comm, +// request); +// } +// catch(conduit::Error e) +// { +// PyErr_SetString(PyExc_Exception, +// e.message().c_str()); +// return NULL; +// } +// +// Py_RETURN_NONE; +// } + + + + +// int CONDUIT_RELAY_API irecv(Node &node, +// int src, +// int tag, +// MPI_Comm comm, +// Request *request); + + +//---------------------------------------------------------------------------// +// conduit::relay::mpi::irecv +//---------------------------------------------------------------------------// +// static PyObject * +// PyRelay_MPI_irecv(PyObject *, //self +// PyObject *args, +// PyObject *kwargs) +// { +// PyObject *py_node = NULL; +// Py_ssize_t source; +// Py_ssize_t tag; +// Py_ssize_t mpi_comm_id; +// PyObject *py_mpi_request = NULL; +// +// // TODO: future also accept mpi4py comm +// +// static const char *kwlist[] = {"node", +// "source", +// "tag", +// "comm", +// "request", +// NULL}; +// +// if (!PyArg_ParseTupleAndKeywords(args, +// kwargs, +// "OnnnO", +// const_cast(kwlist), +// &py_node, +// &source, +// &tag, +// &mpi_comm_id, +// &py_mpi_request)) +// { +// return (NULL); +// } +// +// if(!PyConduit_Node_Check(py_node)) +// { +// PyErr_SetString(PyExc_TypeError, +// "'node' argument must be a " +// "conduit.Node instance"); +// return NULL; +// } +// +// if(!PyRelay_MPI_Request_Check(py_mpi_request)) +// { +// PyErr_SetString(PyExc_TypeError, +// "'request' argument must be a " +// "conduit.relay.mpi Request instance"); +// return NULL; +// } +// +// Node &node = *PyConduit_Node_Get_Node_Ptr(py_node); +// Request *request = PyRelay_MPI_Request_request_pointer((PyRelay_MPI_Request*)py_mpi_request); +// +// // get c mpi comm hnd +// MPI_Comm comm = MPI_Comm_f2c(mpi_comm_id); +// +// try +// { +// relay::mpi::irecv(node, +// source, +// tag, +// comm, +// request); +// } +// catch(conduit::Error e) +// { +// PyErr_SetString(PyExc_Exception, +// e.message().c_str()); +// return NULL; +// } +// +// Py_RETURN_NONE; +// } + + + +// int CONDUIT_RELAY_API wait_send(Request *request, +// MPI_Status *status); +// +// int CONDUIT_RELAY_API wait_recv(Request *request, +// MPI_Status *status); +// +// int CONDUIT_RELAY_API wait_all_send(int count, +// Request requests[], +// MPI_Status statuses[]); +// +// int CONDUIT_RELAY_API wait_all_recv(int count, +// Request requests[], +// MPI_Status statuses[]); +// + +//----------------------------------------------------------------------------- +/// MPI gather (these expect identical schemas) +//----------------------------------------------------------------------------- + +//---------------------------------------------------------------------------// +// conduit::relay::mpi::gather +//---------------------------------------------------------------------------// +static PyObject * +PyRelay_MPI_gather(PyObject *, //self + PyObject *args, + PyObject *kwargs) +{ + PyObject *py_src_node = NULL; + PyObject *py_recv_node = NULL; + Py_ssize_t root; + Py_ssize_t mpi_comm_id; + + // TODO: future also accept mpi4py comm + + static const char *kwlist[] = {"send_node", + "recv_node", + "root", + "comm", + NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, + kwargs, + "OOnn", + const_cast(kwlist), + &py_src_node, + &py_recv_node, + &root, + &mpi_comm_id)) + { + return (NULL); + } + + if(!PyConduit_Node_Check(py_src_node)) + { + PyErr_SetString(PyExc_TypeError, + "'send_node' argument must be a " + "conduit.Node instance"); + return NULL; + } + + if(!PyConduit_Node_Check(py_recv_node)) + { + PyErr_SetString(PyExc_TypeError, + "'recv_node' argument must be a " + "conduit.Node instance"); + return NULL; + } + + Node &send_node = *PyConduit_Node_Get_Node_Ptr(py_src_node); + Node &recv_node = *PyConduit_Node_Get_Node_Ptr(py_recv_node); + + // get c mpi comm hnd + MPI_Comm comm = MPI_Comm_f2c(mpi_comm_id); + + try + { + relay::mpi::gather(send_node, + recv_node, + root, + comm); + } + catch(conduit::Error e) + { + PyErr_SetString(PyExc_Exception, + e.message().c_str()); + return NULL; + } + + Py_RETURN_NONE; +} + +//---------------------------------------------------------------------------// +// conduit::relay::mpi::gather_using_schema +//---------------------------------------------------------------------------// +static PyObject * +PyRelay_MPI_gather_using_schema(PyObject *, //self + PyObject *args, + PyObject *kwargs) +{ + PyObject *py_src_node = NULL; + PyObject *py_recv_node = NULL; + Py_ssize_t root; + Py_ssize_t mpi_comm_id; + + // TODO: future also accept mpi4py comm + + static const char *kwlist[] = {"send_node", + "recv_node", + "root", + "comm", + NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, + kwargs, + "OOnn", + const_cast(kwlist), + &py_src_node, + &py_recv_node, + &root, + &mpi_comm_id)) + { + return (NULL); + } + + if(!PyConduit_Node_Check(py_src_node)) + { + PyErr_SetString(PyExc_TypeError, + "'send_node' argument must be a " + "conduit.Node instance"); + return NULL; + } + + if(!PyConduit_Node_Check(py_recv_node)) + { + PyErr_SetString(PyExc_TypeError, + "'recv_node' argument must be a " + "conduit.Node instance"); + return NULL; + } + + Node &send_node = *PyConduit_Node_Get_Node_Ptr(py_src_node); + Node &recv_node = *PyConduit_Node_Get_Node_Ptr(py_recv_node); + + // get c mpi comm hnd + MPI_Comm comm = MPI_Comm_f2c(mpi_comm_id); + + try + { + relay::mpi::gather_using_schema(send_node, + recv_node, + root, + comm); + } + catch(conduit::Error e) + { + PyErr_SetString(PyExc_Exception, + e.message().c_str()); + return NULL; + } + + Py_RETURN_NONE; +} + +//---------------------------------------------------------------------------// +// conduit::relay::mpi::all_gather +//---------------------------------------------------------------------------// +static PyObject * +PyRelay_MPI_all_gather(PyObject *, //self + PyObject *args, + PyObject *kwargs) +{ + PyObject *py_src_node = NULL; + PyObject *py_recv_node = NULL; + Py_ssize_t mpi_comm_id; + + // TODO: future also accept mpi4py comm + + static const char *kwlist[] = {"send_node", + "recv_node", + "comm", + NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, + kwargs, + "OOn", + const_cast(kwlist), + &py_src_node, + &py_recv_node, + &mpi_comm_id)) + { + return (NULL); + } + + if(!PyConduit_Node_Check(py_src_node)) + { + PyErr_SetString(PyExc_TypeError, + "'send_node' argument must be a " + "conduit.Node instance"); + return NULL; + } + + if(!PyConduit_Node_Check(py_recv_node)) + { + PyErr_SetString(PyExc_TypeError, + "'recv_node' argument must be a " + "conduit.Node instance"); + return NULL; + } + + Node &send_node = *PyConduit_Node_Get_Node_Ptr(py_src_node); + Node &recv_node = *PyConduit_Node_Get_Node_Ptr(py_recv_node); + + // get c mpi comm hnd + MPI_Comm comm = MPI_Comm_f2c(mpi_comm_id); + + try + { + relay::mpi::all_gather(send_node, + recv_node, + comm); + } + catch(conduit::Error e) + { + PyErr_SetString(PyExc_Exception, + e.message().c_str()); + return NULL; + } + + Py_RETURN_NONE; +} + +//---------------------------------------------------------------------------// +// conduit::relay::mpi::gather_using_schema +//---------------------------------------------------------------------------// +static PyObject * +PyRelay_MPI_all_gather_using_schema(PyObject *, //self + PyObject *args, + PyObject *kwargs) +{ + PyObject *py_src_node = NULL; + PyObject *py_recv_node = NULL; + Py_ssize_t mpi_comm_id; + + // TODO: future also accept mpi4py comm + + static const char *kwlist[] = {"send_node", + "recv_node", + "comm", + NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, + kwargs, + "OOn", + const_cast(kwlist), + &py_src_node, + &py_recv_node, + &mpi_comm_id)) + { + return (NULL); + } + + if(!PyConduit_Node_Check(py_src_node)) + { + PyErr_SetString(PyExc_TypeError, + "'send_node' argument must be a " + "conduit.Node instance"); + return NULL; + } + + if(!PyConduit_Node_Check(py_recv_node)) + { + PyErr_SetString(PyExc_TypeError, + "'recv_node' argument must be a " + "conduit.Node instance"); + return NULL; + } + + Node &send_node = *PyConduit_Node_Get_Node_Ptr(py_src_node); + Node &recv_node = *PyConduit_Node_Get_Node_Ptr(py_recv_node); + + // get c mpi comm hnd + MPI_Comm comm = MPI_Comm_f2c(mpi_comm_id); + + try + { + relay::mpi::all_gather_using_schema(send_node, + recv_node, + comm); + } + catch(conduit::Error e) + { + PyErr_SetString(PyExc_Exception, + e.message().c_str()); + return NULL; + } + + Py_RETURN_NONE; +} + +//----------------------------------------------------------------------------- +/// MPI broadcast +//----------------------------------------------------------------------------- + +//---------------------------------------------------------------------------// +// conduit::relay::mpi::broadcast +//---------------------------------------------------------------------------// +static PyObject * +PyRelay_MPI_broadcast(PyObject *, //self + PyObject *args, + PyObject *kwargs) +{ + PyObject *py_node = NULL; + Py_ssize_t root; + Py_ssize_t mpi_comm_id; + + // TODO: future also accept mpi4py comm + + static const char *kwlist[] = {"node", + "root", + "comm", + NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, + kwargs, + "Onn", + const_cast(kwlist), + &py_node, + &root, + &mpi_comm_id)) + { + return (NULL); + } + + if(!PyConduit_Node_Check(py_node)) + { + PyErr_SetString(PyExc_TypeError, + "'node' argument must be a " + "conduit.Node instance"); + return NULL; + } + + Node &node = *PyConduit_Node_Get_Node_Ptr(py_node); + + // get c mpi comm hnd + MPI_Comm comm = MPI_Comm_f2c(mpi_comm_id); + + try + { + relay::mpi::broadcast(node, root, comm); + } + catch(conduit::Error e) + { + PyErr_SetString(PyExc_Exception, + e.message().c_str()); + return NULL; + } + + Py_RETURN_NONE; +} + +//---------------------------------------------------------------------------// +// conduit::relay::mpi::broadcast_using_schema +//---------------------------------------------------------------------------// +static PyObject * +PyRelay_MPI_broadcast_using_schema(PyObject *, //self + PyObject *args, + PyObject *kwargs) +{ + PyObject *py_node = NULL; + Py_ssize_t root; + Py_ssize_t mpi_comm_id; + + // TODO: future also accept mpi4py comm + + static const char *kwlist[] = {"node", + "root", + "comm", + NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, + kwargs, + "Onn", + const_cast(kwlist), + &py_node, + &root, + &mpi_comm_id)) + { + return (NULL); + } + + if(!PyConduit_Node_Check(py_node)) + { + PyErr_SetString(PyExc_TypeError, + "'node' argument must be a " + "conduit.Node instance"); + return NULL; + } + + Node &node = *PyConduit_Node_Get_Node_Ptr(py_node); + + // get c mpi comm hnd + MPI_Comm comm = MPI_Comm_f2c(mpi_comm_id); + + try + { + relay::mpi::broadcast_using_schema(node, root, comm); + } + catch(conduit::Error e) + { + PyErr_SetString(PyExc_Exception, + e.message().c_str()); + return NULL; + } + + Py_RETURN_NONE; +} + + +//---------------------------------------------------------------------------// +// Python Module Method Defs +//---------------------------------------------------------------------------// + +// TODO: Future Support ? +// reduce (generic, reduce op as arg) +// all_reduce (generic, reduce op as arg) + +// isend +// irecv +// wait_send +// wait_recv +// wait_all_send +// wait_all_recv +static PyMethodDef relay_mpi_python_funcs[] = +{ + //-----------------------------------------------------------------------// + //-----------------------------------------------------------------------// + {"about", + (PyCFunction)PyRelay_MPI_about, + METH_NOARGS, + "About Relay MPI"}, + {"rank", + (PyCFunction)PyRelay_MPI_rank, + METH_VARARGS | METH_KEYWORDS, + "MPI Comm Rank"}, + {"size", + (PyCFunction)PyRelay_MPI_size, + METH_VARARGS | METH_KEYWORDS, + "MPI Comm Size"}, + // -- send + recv --- + {"send", + (PyCFunction)PyRelay_MPI_send, + METH_VARARGS | METH_KEYWORDS, + "Send Conduit Node via MPI Send"}, + {"recv", + (PyCFunction)PyRelay_MPI_recv, + METH_VARARGS | METH_KEYWORDS, + "Receive Conduit Node via MPI Recv"}, + {"send_using_schema", + (PyCFunction)PyRelay_MPI_send_using_schema, + METH_VARARGS | METH_KEYWORDS, + "Send Conduit Node and Schema via MPI Send"}, + {"recv_using_schema", + (PyCFunction)PyRelay_MPI_recv_using_schema, + METH_VARARGS | METH_KEYWORDS, + "Receive Conduit Node and Schema via MPI Recv"}, + // -- reduce -- + {"sum_reduce", + (PyCFunction)PyRelay_MPI_sum_reduce, + METH_VARARGS | METH_KEYWORDS, + "Sum Reduce a Conduit Node"}, + {"min_reduce", + (PyCFunction)PyRelay_MPI_min_reduce, + METH_VARARGS | METH_KEYWORDS, + "Min Reduce a Conduit Node"}, + {"max_reduce", + (PyCFunction)PyRelay_MPI_max_reduce, + METH_VARARGS | METH_KEYWORDS, + "Max Reduce a Conduit Node"}, + {"prod_reduce", + (PyCFunction)PyRelay_MPI_prod_reduce, + METH_VARARGS | METH_KEYWORDS, + "Prod Reduce a Conduit Node"}, + // -- all reduce -- + {"sum_all_reduce", + (PyCFunction)PyRelay_MPI_sum_all_reduce, + METH_VARARGS | METH_KEYWORDS, + "Sum All Reduce a Conduit Node"}, + {"min_all_reduce", + (PyCFunction)PyRelay_MPI_min_all_reduce, + METH_VARARGS | METH_KEYWORDS, + "Min All Reduce a Conduit Node"}, + {"max_all_reduce", + (PyCFunction)PyRelay_MPI_max_all_reduce, + METH_VARARGS | METH_KEYWORDS, + "Max All Reduce a Conduit Node"}, + {"prod_all_reduce", + (PyCFunction)PyRelay_MPI_prod_all_reduce, + METH_VARARGS | METH_KEYWORDS, + "Prod All Reduce a Conduit Node"}, + // -- gather -- + {"gather", + (PyCFunction)PyRelay_MPI_gather, + METH_VARARGS | METH_KEYWORDS, + "MPI Gather using Conduit Nodes"}, + {"all_gather", + (PyCFunction)PyRelay_MPI_all_gather, + METH_VARARGS | METH_KEYWORDS, + "MPI All Gather using Conduit Nodes"}, + {"gather_using_schema", + (PyCFunction)PyRelay_MPI_gather_using_schema, + METH_VARARGS | METH_KEYWORDS, + "MPI Gather using Conduit Nodes and their Schemas"}, + {"all_gather_using_schema", + (PyCFunction)PyRelay_MPI_all_gather_using_schema, + METH_VARARGS | METH_KEYWORDS, + "MPI All Gather using Conduit Nodes and their Schemas"}, + // -- broadcast -- + {"broadcast", + (PyCFunction)PyRelay_MPI_broadcast, + METH_VARARGS | METH_KEYWORDS, + "MPI Broadcast a Conduit Node"}, + {"broadcast_using_schema", + (PyCFunction)PyRelay_MPI_broadcast_using_schema, + METH_VARARGS | METH_KEYWORDS, + "MPI Broadcast a Conduit Node and its Schema"}, + // -- isend + irecv --- + // {"isend", + // (PyCFunction)PyRelay_MPI_send, + // METH_VARARGS | METH_KEYWORDS, + // "Send Conduit Node via MPI ISend"}, + // {"irecv", + // (PyCFunction)PyRelay_MPI_send, + // METH_VARARGS | METH_KEYWORDS, + // "Receive Conduit Node via MPI IRecv"}, + //-----------------------------------------------------------------------// + // end relay mpi methods table + //-----------------------------------------------------------------------// + {NULL, NULL, METH_VARARGS, NULL} +}; + +//---------------------------------------------------------------------------// +//---------------------------------------------------------------------------// +// Module Init Code +//---------------------------------------------------------------------------// +//---------------------------------------------------------------------------// + +struct module_state { + PyObject *error; +}; + +//---------------------------------------------------------------------------// +#if defined(IS_PY3K) +#define GETSTATE(m) ((struct module_state*)PyModule_GetState(m)) +#else +#define GETSTATE(m) (&_state) +static struct module_state _state; +#endif +//---------------------------------------------------------------------------// + +//---------------------------------------------------------------------------// +// Extra Module Setup Logic for Python3 +//---------------------------------------------------------------------------// +#if defined(IS_PY3K) +//---------------------------------------------------------------------------// +static int +relay_mpi_python_traverse(PyObject *m, visitproc visit, void *arg) +{ + Py_VISIT(GETSTATE(m)->error); + return 0; +} + +//---------------------------------------------------------------------------// +static int +relay_mpi_python_clear(PyObject *m) +{ + Py_CLEAR(GETSTATE(m)->error); + return 0; +} + +//---------------------------------------------------------------------------// +static struct PyModuleDef relay_mpi_python_module_def = +{ + PyModuleDef_HEAD_INIT, + "relay_mpi_python", + NULL, + sizeof(struct module_state), + relay_mpi_python_funcs, + NULL, + relay_mpi_python_traverse, + relay_mpi_python_clear, + NULL +}; + + + +#endif + +//---------------------------------------------------------------------------// +// The module init function signature is different between py2 and py3 +// This macro simplifies the process of returning when an init error occurs. +//---------------------------------------------------------------------------// +#if defined(IS_PY3K) +#define PY_MODULE_INIT_RETURN_ERROR return NULL +#else +#define PY_MODULE_INIT_RETURN_ERROR return +#endif +//---------------------------------------------------------------------------// + + +//---------------------------------------------------------------------------// +// Main entry point +//---------------------------------------------------------------------------// +extern "C" +//---------------------------------------------------------------------------// +#if defined(IS_PY3K) +CONDUIT_RELAY_PYTHON_API PyObject * PyInit_conduit_relay_mpi_python(void) +#else +CONDUIT_RELAY_PYTHON_API void initconduit_relay_mpi_python(void) +#endif +//---------------------------------------------------------------------------// +{ + //-----------------------------------------------------------------------// + // create our main module + //-----------------------------------------------------------------------// + +#if defined(IS_PY3K) + PyObject *relay_mpi_module = PyModule_Create(&relay_mpi_python_module_def); +#else + PyObject *relay_mpi_module = Py_InitModule((char*)"conduit_relay_mpi_python", + relay_mpi_python_funcs); +#endif + + + if(relay_mpi_module == NULL) + { + PY_MODULE_INIT_RETURN_ERROR; + } + + struct module_state *st = GETSTATE(relay_mpi_module); + + st->error = PyErr_NewException((char*)"relay_mpi_python.Error", + NULL, + NULL); + if (st->error == NULL) + { + Py_DECREF(relay_mpi_module); + PY_MODULE_INIT_RETURN_ERROR; + } + + // setup for conduit python c api + if(import_conduit() < 0) + { + PY_MODULE_INIT_RETURN_ERROR; + } + //-----------------------------------------------------------------------// + // init our custom types + //-----------------------------------------------------------------------// + + if (PyType_Ready(&PyRelay_MPI_Request_TYPE) < 0) + { + PY_MODULE_INIT_RETURN_ERROR; + } + + //-----------------------------------------------------------------------// + // add Request + //-----------------------------------------------------------------------// + Py_INCREF(&PyRelay_MPI_Request_TYPE); + PyModule_AddObject(relay_mpi_module, + "Request", + (PyObject*)&PyRelay_MPI_Request_TYPE); + +#if defined(IS_PY3K) + return relay_mpi_module; +#endif + +} + diff --git a/src/libs/relay/python/conduit_relay_python_exports.h.in b/src/libs/relay/python/conduit_relay_python_exports.h.in index 089e5dd80..6913d66ff 100644 --- a/src/libs/relay/python/conduit_relay_python_exports.h.in +++ b/src/libs/relay/python/conduit_relay_python_exports.h.in @@ -63,6 +63,8 @@ defined(conduit_relay_python_EXPORTS) || \ defined(CONDUIT_RELAY_IO_PYTHON_EXPORTS) || \ defined(conduit_relay_io_python_EXPORTS) || \ + defined(CONDUIT_RELAY_MPI_PYTHON_EXPORTS) || \ + defined(conduit_relay_mpi_python_EXPORTS) || \ defined(CONDUIT_RELAY_WEB_PYTHON_EXPORTS)|| \ defined(conduit_relay_web_python_EXPORTS) /* define catch all def */ diff --git a/src/libs/relay/python/py_src/__init__.py b/src/libs/relay/python/py_src/__init__.py index d00775ebb..9c36e1fc7 100644 --- a/src/libs/relay/python/py_src/__init__.py +++ b/src/libs/relay/python/py_src/__init__.py @@ -50,6 +50,7 @@ from . import io from . import web +from . import mpi diff --git a/src/libs/relay/python/py_src/mpi/__init__.py b/src/libs/relay/python/py_src/mpi/__init__.py new file mode 100644 index 000000000..4e3c23173 --- /dev/null +++ b/src/libs/relay/python/py_src/mpi/__init__.py @@ -0,0 +1,53 @@ +############################################################################### +# Copyright (c) 2014-2020, Lawrence Livermore National Security, LLC. +# +# Produced at the Lawrence Livermore National Laboratory +# +# LLNL-CODE-666778 +# +# All rights reserved. +# +# This file is part of Conduit. +# +# For details, see: http://software.llnl.gov/conduit/. +# +# Please also read conduit/LICENSE +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the disclaimer below. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the disclaimer (as noted below) in the +# documentation and/or other materials provided with the distribution. +# +# * Neither the name of the LLNS/LLNL nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL SECURITY, +# LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING +# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +############################################################################### + +############################################################################### +# file: __init__.py +# Purpose: Main init for the conduit relay mpi module. +############################################################################### +from .conduit_relay_mpi_python import * + + + + diff --git a/src/libs/relay/python/setup.py b/src/libs/relay/python/setup.py index e97afdfe9..c5d1f7874 100644 --- a/src/libs/relay/python/setup.py +++ b/src/libs/relay/python/setup.py @@ -55,6 +55,7 @@ def run(self): package_dir = {'conduit.relay':'py_src'}, packages=['conduit.relay', 'conduit.relay.io', + 'conduit.relay.mpi', 'conduit.relay.web'], cmdclass={'install_egg_info': SkipEggInfo}) diff --git a/src/tests/docs/CMakeLists.txt b/src/tests/docs/CMakeLists.txt index 67dec7007..3910215f8 100644 --- a/src/tests/docs/CMakeLists.txt +++ b/src/tests/docs/CMakeLists.txt @@ -71,6 +71,9 @@ set(DOCS_PYTHON_TESTS t_conduit_docs_tutorial_python_ownership t_conduit_docs_tutorial_python_relay_io_handle_examples) +set(DOCS_PYTHON_MPI_TESTS + t_conduit_docs_tutorial_python_relay_mpi_examples) + ################################ # Add our tests @@ -101,6 +104,15 @@ if(PYTHON_FOUND) add_python_test(TEST ${TEST} FOLDER tests/docs) endforeach() + + if(MPI_FOUND) + foreach(TEST ${DOCS_PYTHON_MPI_TESTS}) + add_python_mpi_test(TEST ${TEST} + NUM_MPI_TASKS 2) + # FOLDER tests/docs) + endforeach() + endif() + else() message(STATUS "Python disabled: Skipping conduit python tutorial examples") endif() diff --git a/src/tests/docs/t_conduit_docs_tutorial_python_relay_mpi_examples.py b/src/tests/docs/t_conduit_docs_tutorial_python_relay_mpi_examples.py new file mode 100644 index 000000000..f5643cf91 --- /dev/null +++ b/src/tests/docs/t_conduit_docs_tutorial_python_relay_mpi_examples.py @@ -0,0 +1,299 @@ +############################################################################### +# Copyright (c) 2014-2019, Lawrence Livermore National Security, LLC. +# +# Produced at the Lawrence Livermore National Laboratory +# +# LLNL-CODE-666778 +# +# All rights reserved. +# +# This file is part of Conduit. +# +# For details, see: http://software.llnl.gov/conduit/. +# +# Please also read conduit/LICENSE +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the disclaimer below. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the disclaimer (as noted below) in the +# documentation and/or other materials provided with the distribution. +# +# * Neither the name of the LLNS/LLNL nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL SECURITY, +# LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING +# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +############################################################################### +""" + file: t_conduit_docs_tutorial_python_relay_mpi_examples.py +""" + +import sys +import unittest +import inspect +import numpy + + + +def BEGIN_EXAMPLE(tag): + from mpi4py import MPI + MPI.COMM_WORLD.Barrier() + if MPI.COMM_WORLD.rank == 0: + print('\nBEGIN_EXAMPLE("' + tag + '")') + MPI.COMM_WORLD.Barrier() + +def END_EXAMPLE(tag): + from mpi4py import MPI + MPI.COMM_WORLD.Barrier() + if MPI.COMM_WORLD.rank == 0: + print('\nEND_EXAMPLE("' + tag + '")') + MPI.COMM_WORLD.Barrier() + +class Conduit_Tutorial_Python_Relay_IO_Handle(unittest.TestCase): + + def test_001_mpi_send_and_recv_using_schema(self): + # skip tests on windows until we work out proper + # mpi4py install for our windows ci + if sys.platform == "win32": + return + BEGIN_EXAMPLE("py_mpi_send_and_recv_using_schema") + import conduit + import conduit.relay as relay + import conduit.relay.mpi + from mpi4py import MPI + + # Note: example expects 2 mpi tasks + + # get a comm id from mpi4py world comm + comm_id = MPI.COMM_WORLD.py2f() + # get our rank and the comm's size + comm_rank = relay.mpi.rank(comm_id) + comm_size = relay.mpi.size(comm_id) + + # send a node and its schema from rank 0 to rank 1 + n = conduit.Node() + if comm_rank == 0: + # setup node to send on rank 0 + n["a/data"] = 1.0 + n["a/more_data"] = 2.0 + n["a/b/my_string"] = "value" + + # show node data on rank 0 + if comm_rank == 0: + print("[rank: {}] sending: {}".format(comm_rank,n.to_yaml())) + + if comm_rank == 0: + relay.mpi.send_using_schema(n,dest=1,tag=0,comm=comm_id) + else: + relay.mpi.recv_using_schema(n,source=0,tag=0,comm=comm_id) + + # show received node data on rank 1 + if comm_rank == 1: + print("[rank: {}] received: {}".format(comm_rank,n.to_yaml())) + + END_EXAMPLE("py_mpi_send_and_recv_using_schema") + + + def test_002_mpi_send_and_recv(self): + # skip tests on windows until we work out proper + # mpi4py install for our windows ci + if sys.platform == "win32": + return + BEGIN_EXAMPLE("py_mpi_send_and_recv") + import conduit + import conduit.relay as relay + import conduit.relay.mpi + from mpi4py import MPI + + # Note: example expects 2 mpi tasks + + # get a comm id from mpi4py world comm + comm_id = MPI.COMM_WORLD.py2f() + # get our rank and the comm's size + comm_rank = relay.mpi.rank(comm_id) + comm_size = relay.mpi.size(comm_id) + + # send data from a node on rank 0 to rank 1 + # (both ranks have nodes with compatible schemas) + n = conduit.Node(conduit.DataType.int64(4)) + if comm_rank == 0: + # setup node to send on rank 0 + vals = n.value() + for i in range(4): + vals[i] = i * i + + # show node data on rank 0 + if comm_rank == 0: + print("[rank: {}] sending: {}".format(comm_rank,n.to_yaml())) + + if comm_rank == 0: + relay.mpi.send(n,dest=1,tag=0,comm=comm_id) + else: + relay.mpi.recv(n,source=0,tag=0,comm=comm_id) + + # show received node data on rank 1 + if comm_rank == 1: + print("[rank: {}] received: {}".format(comm_rank,n.to_yaml())) + + END_EXAMPLE("py_mpi_send_and_recv") + + def test_003_mpi_bcast_using_schema(self): + # skip tests on windows until we work out proper + # mpi4py install for our windows ci + if sys.platform == "win32": + return + BEGIN_EXAMPLE("py_mpi_bcast_using_schema") + import conduit + import conduit.relay as relay + import conduit.relay.mpi + from mpi4py import MPI + + # Note: example expects 2 mpi tasks + + # get a comm id from mpi4py world comm + comm_id = MPI.COMM_WORLD.py2f() + # get our rank and the comm's size + comm_rank = relay.mpi.rank(comm_id) + comm_size = relay.mpi.size(comm_id) + + # send a node and its schema from rank 0 to rank 1 + n = conduit.Node() + if comm_rank == 0: + # setup node to broadcast on rank 0 + n["a/data"] = 1.0 + n["a/more_data"] = 2.0 + n["a/b/my_string"] = "value" + + # show node data on rank 0 + if comm_rank == 0: + print("[rank: {}] broadcasting: {}".format(comm_rank,n.to_yaml())) + + relay.mpi.broadcast_using_schema(n,root=0,comm=comm_id) + + # show received node data on rank 1 + if comm_rank == 1: + print("[rank: {}] received: {}".format(comm_rank,n.to_yaml())) + + END_EXAMPLE("py_mpi_bcast_using_schema") + + def test_004_mpi_bcast_using_schema(self): + # skip tests on windows until we work out proper + # mpi4py install for our windows ci + if sys.platform == "win32": + return + BEGIN_EXAMPLE("py_mpi_bcast") + import conduit + import conduit.relay as relay + import conduit.relay.mpi + from mpi4py import MPI + + # Note: example expects 2 mpi tasks + + # get a comm id from mpi4py world comm + comm_id = MPI.COMM_WORLD.py2f() + # get our rank and the comm's size + comm_rank = relay.mpi.rank(comm_id) + comm_size = relay.mpi.size(comm_id) + + # send data from a node on rank 0 to rank 1 + # (both ranks have nodes with compatible schemas) + n = conduit.Node(conduit.DataType.int64(4)) + if comm_rank == 0: + # setup node to send on rank 0 + vals = n.value() + for i in range(4): + vals[i] = i * i + + # show node data on rank 0 + if comm_rank == 0: + print("[rank: {}] broadcasting: {}".format(comm_rank,n.to_yaml())) + + relay.mpi.broadcast_using_schema(n,root=0,comm=comm_id) + + # show received node data on rank 1 + if comm_rank == 1: + print("[rank: {}] received: {}".format(comm_rank,n.to_yaml())) + + END_EXAMPLE("py_mpi_bcast") + + def test_005_mpi_sum_all_reduce(self): + # skip tests on windows until we work out proper + # mpi4py install for our windows ci + if sys.platform == "win32": + return + BEGIN_EXAMPLE("py_mpi_sum_all_reduce") + import conduit + import conduit.relay as relay + import conduit.relay.mpi + from mpi4py import MPI + + # get a comm id from mpi4py world comm + comm_id = MPI.COMM_WORLD.py2f() + # get our rank and the comm's size + comm_rank = relay.mpi.rank(comm_id) + comm_size = relay.mpi.size(comm_id) + + # gather data all ranks + # (ranks have nodes with compatible schemas) + n = conduit.Node(conduit.DataType.int64(4)) + n_res = conduit.Node(conduit.DataType.int64(4)) + # data to reduce + vals = n.value() + for i in range(4): + vals[i] = 1 + + relay.mpi.sum_all_reduce(n,n_res,comm=comm_id) + # answer should be an array with each value == comm_size + # show result on rank 0 + if comm_rank == 0: + print("[rank: {}] sum reduce result: {}".format(comm_rank,n_res.to_yaml())) + + END_EXAMPLE("py_mpi_sum_all_reduce") + + def test_006_mpi_all_gather(self): + # skip tests on windows until we work out proper + # mpi4py install for our windows ci + if sys.platform == "win32": + return + BEGIN_EXAMPLE("py_mpi_all_gather_using_schema") + import conduit + import conduit.relay as relay + import conduit.relay.mpi + from mpi4py import MPI + + # get a comm id from mpi4py world comm + comm_id = MPI.COMM_WORLD.py2f() + # get our rank and the comm's size + comm_rank = relay.mpi.rank(comm_id) + comm_size = relay.mpi.size(comm_id) + + n = conduit.Node(conduit.DataType.int64(4)) + n_res = conduit.Node() + # data to gather + vals = n.value() + for i in range(4): + vals[i] = comm_rank + + relay.mpi.all_gather_using_schema(n,n_res,comm=comm_id) + # show result on rank 0 + if comm_rank == 0: + print("[rank: {}] all gather using schema result: {}".format(comm_rank,n_res.to_yaml())) + + END_EXAMPLE("py_mpi_all_gather_using_schema") diff --git a/src/tests/relay/python/CMakeLists.txt b/src/tests/relay/python/CMakeLists.txt index 4e27ec71e..4b88fce0f 100644 --- a/src/tests/relay/python/CMakeLists.txt +++ b/src/tests/relay/python/CMakeLists.txt @@ -50,9 +50,18 @@ set(PYTHON_MODULE_TESTS t_python_relay_smoke t_python_relay_io_handle t_python_relay_web) +set(PYTHON_MODULE_MPI_TESTS t_python_relay_mpi) foreach(TEST ${PYTHON_MODULE_TESTS}) add_python_test(TEST ${TEST} FOLDER tests/relay/python) endforeach() +if(MPI_FOUND) + foreach(TEST ${PYTHON_MODULE_MPI_TESTS}) + add_python_mpi_test(TEST ${TEST} + NUM_MPI_TASKS 2) + # FOLDER tests/relay/python) + endforeach() +endif() + diff --git a/src/tests/relay/python/t_python_relay_mpi.py b/src/tests/relay/python/t_python_relay_mpi.py new file mode 100644 index 000000000..67187e075 --- /dev/null +++ b/src/tests/relay/python/t_python_relay_mpi.py @@ -0,0 +1,430 @@ +############################################################################### +# Copyright (c) 2015-2020, Lawrence Livermore National Security, LLC. +# +# Produced at the Lawrence Livermore National Laboratory +# +# LLNL-CODE-716457 +# +# All rights reserved. +# +# This file is part of Ascent. +# +# For details, see: http://ascent.readthedocs.io/. +# +# Please also read ascent/LICENSE +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the disclaimer below. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the disclaimer (as noted below) in the +# documentation and/or other materials provided with the distribution. +# +# * Neither the name of the LLNS/LLNL nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL SECURITY, +# LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING +# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +############################################################################### + + +""" + file: t_python_relay_mpi.py + description: Unit tests for relay mpi + +""" + +import sys +import unittest +import os +import math + +import numpy as np + +import conduit +import conduit.blueprint +import conduit.relay as relay +import conduit.relay.mpi + +# TODO: +# from mpi4py import MPI + +class Test_Relay_MPI_Module(unittest.TestCase): + + def test_about(self): + # skip tests on windows until we work out proper + # mpi4py install for our windows ci + if sys.platform == "win32": + return + from mpi4py import MPI + print(relay.mpi.about()) + self.assertTrue(True) + + def test_rank_and_size(self): + # skip tests on windows until we work out proper + # mpi4py install for our windows ci + if sys.platform == "win32": + return + from mpi4py import MPI + comm_id = MPI.COMM_WORLD.py2f() + rank = relay.mpi.rank(comm_id) + size = relay.mpi.size(comm_id) + self.assertEqual(rank,MPI.COMM_WORLD.rank) + self.assertEqual(size,MPI.COMM_WORLD.size) + + def test_send_recv_using_schema(self): + # skip tests on windows until we work out proper + # mpi4py install for our windows ci + if sys.platform == "win32": + return + from mpi4py import MPI + n = conduit.Node() + comm_id = MPI.COMM_WORLD.py2f() + rank = relay.mpi.rank(comm_id) + in_vals = np.zeros(3,dtype=np.float64) + + if rank == 0: + in_vals[0] = rank + 1; + in_vals[1] = 3.4124 * rank + in_vals[2] = 10.7 - rank + n.set(in_vals); + relay.mpi.send_using_schema(n,1,0,comm_id); + else: + relay.mpi.recv_using_schema(n,0,0,comm_id); + + res_vals = n.value(); + self.assertEqual(res_vals[0], 1) + self.assertEqual(res_vals[1], 0) + self.assertEqual(res_vals[2], 10.7) + n.reset() + if rank == 0: + n["value/a"] = 1; + n["value/b"] = 2; + relay.mpi.send_using_schema(n,1,0,comm_id); + else: + relay.mpi.recv_using_schema(n,0,0,comm_id); + + val_a = n["value/a"] + val_b = n["value/b"] + + self.assertEqual(val_a, 1) + self.assertEqual(val_b, 2) + + def test_send_recv_without_using_schema(self): + # skip tests on windows until we work out proper + # mpi4py install for our windows ci + if sys.platform == "win32": + return + from mpi4py import MPI + n = conduit.Node() + comm_id = MPI.COMM_WORLD.py2f() + rank = relay.mpi.rank(comm_id) + vals = np.zeros(3,dtype=np.float64) + n.set_external(vals); + print(rank) + if rank == 0: + vals[0] = rank + 1 + vals[1] = 3.4124 * rank + vals[2] = 10.7 - rank + relay.mpi.send(n,1,0,comm_id); + else: + relay.mpi.recv(n,0,0,comm_id); + + res_vals = n.value(); + self.assertEqual(res_vals[0], 1) + self.assertEqual(res_vals[1], 0) + self.assertEqual(res_vals[2], 10.7) + + def test_reduce_helpers(self): + # skip tests on windows until we work out proper + # mpi4py install for our windows ci + if sys.platform == "win32": + return + from mpi4py import MPI + comm_id = MPI.COMM_WORLD.py2f() + comm_rank = relay.mpi.rank(comm_id) + comm_size = relay.mpi.size(comm_id) + snd = conduit.Node(conduit.DataType.int64(5)) + rcv = conduit.Node(conduit.DataType.int64(5)) + + snd_vals = snd.value() + rcv_vals = rcv.value() + + # sum + print("sum") + for i in range(5): + snd_vals[i] = 10 + relay.mpi.sum_reduce(snd, rcv, 0, comm_id); + if comm_rank == 0: + print(rcv_vals) + for i in range(5): + self.assertEqual(rcv_vals[i], 10 * comm_size) + + # prod + print("prod") + for i in range(5): + snd_vals[i] = 2 + relay.mpi.prod_reduce(snd, rcv, 0, comm_id) + if comm_rank == 0: + print(rcv_vals) + for i in range(5): + self.assertEqual(rcv_vals[i], math.pow(comm_size,2) ) + + # max + print("max") + for i in range(5): + snd_vals[i] = comm_rank * 10 +1 + relay.mpi.max_reduce(snd, rcv, 0, comm_id) + if comm_rank == 0: + print(rcv_vals) + for i in range(5): + self.assertEqual(rcv_vals[i], 10 * (comm_size-1) + 1 ) + + # min + print("min") + for i in range(5): + snd_vals[i] = comm_rank * 10 +1 + relay.mpi.min_reduce(snd, rcv, 0, comm_id) + if comm_rank == 0: + print(rcv_vals) + for i in range(5): + self.assertEqual(rcv_vals[i], 1) + + def test_all_reduce_helpers(self): + # skip tests on windows until we work out proper + # mpi4py install for our windows ci + if sys.platform == "win32": + return + from mpi4py import MPI + comm_id = MPI.COMM_WORLD.py2f() + comm_rank = relay.mpi.rank(comm_id) + comm_size = relay.mpi.size(comm_id) + snd = conduit.Node(conduit.DataType.int64(5)) + rcv = conduit.Node(conduit.DataType.int64(5)) + + snd_vals = snd.value() + rcv_vals = rcv.value() + + # sum + print("sum") + for i in range(5): + snd_vals[i] = 10 + relay.mpi.sum_all_reduce(snd, rcv, comm_id); + print(rcv_vals) + for i in range(5): + self.assertEqual(rcv_vals[i], 10 * comm_size) + + # prod + print("prod") + for i in range(5): + snd_vals[i] = 2 + relay.mpi.prod_all_reduce(snd, rcv, comm_id) + print(rcv_vals) + for i in range(5): + self.assertEqual(rcv_vals[i], math.pow(comm_size,2) ) + + # max + print("max") + for i in range(5): + snd_vals[i] = comm_rank * 10 +1 + relay.mpi.max_all_reduce(snd, rcv, comm_id) + print(rcv_vals) + for i in range(5): + self.assertEqual(rcv_vals[i], 10 * (comm_size-1) + 1 ) + + # min + print("min") + for i in range(5): + snd_vals[i] = comm_rank * 10 +1 + relay.mpi.min_all_reduce(snd, rcv, comm_id) + rcv_vals = rcv.value() + print(rcv_vals) + for i in range(5): + self.assertEqual(rcv_vals[i], 1) + + def test_gather_simple(self): + # skip tests on windows until we work out proper + # mpi4py install for our windows ci + if sys.platform == "win32": + return + from mpi4py import MPI + comm_id = MPI.COMM_WORLD.py2f() + comm_rank = relay.mpi.rank(comm_id) + comm_size = relay.mpi.size(comm_id) + n = conduit.Node() + n["values/a"] = comm_rank+1; + n["values/b"] = comm_rank+2; + n["values/c"] = comm_rank+3; + + rcv = conduit.Node() + relay.mpi.gather(n,rcv,0,comm_id) + if comm_rank == 0: + print(rcv) + self.assertEqual(rcv[0]["values/a"],1) + self.assertEqual(rcv[0]["values/b"],2) + self.assertEqual(rcv[0]["values/c"],3) + self.assertEqual(rcv[1]["values/a"],2) + self.assertEqual(rcv[1]["values/b"],3) + self.assertEqual(rcv[1]["values/c"],4) + + def test_all_gather_simple(self): + # skip tests on windows until we work out proper + # mpi4py install for our windows ci + if sys.platform == "win32": + return + from mpi4py import MPI + comm_id = MPI.COMM_WORLD.py2f() + comm_rank = relay.mpi.rank(comm_id) + comm_size = relay.mpi.size(comm_id) + n = conduit.Node() + n["values/a"] = comm_rank+1; + n["values/b"] = comm_rank+2; + n["values/c"] = comm_rank+3; + + rcv = conduit.Node() + relay.mpi.all_gather(n,rcv,comm_id) + print(rcv) + self.assertEqual(rcv[0]["values/a"],1) + self.assertEqual(rcv[0]["values/b"],2) + self.assertEqual(rcv[0]["values/c"],3) + self.assertEqual(rcv[1]["values/a"],2) + self.assertEqual(rcv[1]["values/b"],3) + self.assertEqual(rcv[1]["values/c"],4) + + def test_gather_using_schema_simple(self): + # skip tests on windows until we work out proper + # mpi4py install for our windows ci + if sys.platform == "win32": + return + from mpi4py import MPI + comm_id = MPI.COMM_WORLD.py2f() + comm_rank = relay.mpi.rank(comm_id) + comm_size = relay.mpi.size(comm_id) + n = conduit.Node() + n["values/a"] = comm_rank+1; + n["values/b"] = comm_rank+2; + n["values/c"] = comm_rank+3; + + rcv = conduit.Node() + relay.mpi.gather_using_schema(n,rcv,0,comm_id) + if comm_rank == 0: + print(rcv) + self.assertEqual(rcv[0]["values/a"],1) + self.assertEqual(rcv[0]["values/b"],2) + self.assertEqual(rcv[0]["values/c"],3) + self.assertEqual(rcv[1]["values/a"],2) + self.assertEqual(rcv[1]["values/b"],3) + self.assertEqual(rcv[1]["values/c"],4) + + def test_all_gather_using_schema_simple(self): + # skip tests on windows until we work out proper + # mpi4py install for our windows ci + if sys.platform == "win32": + return + from mpi4py import MPI + comm_id = MPI.COMM_WORLD.py2f() + comm_rank = relay.mpi.rank(comm_id) + comm_size = relay.mpi.size(comm_id) + n = conduit.Node() + n["values/a"] = comm_rank+1; + n["values/b"] = comm_rank+2; + n["values/c"] = comm_rank+3; + + rcv = conduit.Node() + relay.mpi.all_gather_using_schema(n,rcv,comm_id) + print(rcv) + self.assertEqual(rcv[0]["values/a"],1) + self.assertEqual(rcv[0]["values/b"],2) + self.assertEqual(rcv[0]["values/c"],3) + self.assertEqual(rcv[1]["values/a"],2) + self.assertEqual(rcv[1]["values/b"],3) + self.assertEqual(rcv[1]["values/c"],4) + + def test_bcast(self): + # skip tests on windows until we work out proper + # mpi4py install for our windows ci + if sys.platform == "win32": + return + from mpi4py import MPI + comm_id = MPI.COMM_WORLD.py2f() + comm_rank = relay.mpi.rank(comm_id) + comm_size = relay.mpi.size(comm_id) + + for root in range(comm_size): + n = conduit.Node() + n.set(conduit.DataType.int64(3)) + vals = n.value() + if comm_rank == root: + vals[0] = 11 + vals[1] = 22 + vals[2] = 33 + relay.mpi.broadcast(n,root,comm_id) + print(n) + self.assertEqual(vals[0],11) + self.assertEqual(vals[1],22) + self.assertEqual(vals[2],33) + + for root in range(comm_size): + n = conduit.Node() + if comm_rank == root: + n["a/b/c/d/e/f"].set(np.int64(10)) + else: + n["a/b/c/d/e/f"].set(np.int64(0)) + relay.mpi.broadcast(n,root,comm_id) + val = n["a/b/c/d/e/f"] + self.assertEqual(val,10) + + def test_bcast_using_schema(self): + # skip tests on windows until we work out proper + # mpi4py install for our windows ci + if sys.platform == "win32": + return + from mpi4py import MPI + comm_id = MPI.COMM_WORLD.py2f() + comm_rank = relay.mpi.rank(comm_id) + comm_size = relay.mpi.size(comm_id) + + for root in range(comm_size): + n = conduit.Node() + if comm_rank == root: + n.set(conduit.DataType.int64(3)) + vals = n.value() + vals[0] = 11 + vals[1] = 22 + vals[2] = 33 + relay.mpi.broadcast_using_schema(n,root,comm_id) + print(n) + vals = n.value() + self.assertEqual(vals[0],11) + self.assertEqual(vals[1],22) + self.assertEqual(vals[2],33) + + for root in range(comm_size): + n = conduit.Node() + if comm_rank == root: + n["a/b/c/d/e/f"].set(np.int64(10)) + relay.mpi.broadcast_using_schema(n,root,comm_id) + val = n["a/b/c/d/e/f"] + self.assertEqual(val,10) + + +if __name__ == '__main__': + unittest.main() + + + From 8b4da06b7fa675fbdb73b505490142e5eae804ea Mon Sep 17 00:00:00 2001 From: Noah Elliott Date: Mon, 20 Apr 2020 11:08:23 -0700 Subject: [PATCH 2/5] verify for structured options for adjsets (#374) * Verify method for blueprint nestset * Updated the 'blueprint::mesh::nestset' verify function based on discussed refinements (circa 5/29). * Adding index verifiers for 'nestsets' Blueprint field. * Finished the initial implementation of the misc nestsets blueprint example. * * Added all of the test cases for the new structured AMR schema. * Made some minor improvements to the automated structured AMR schema generation code. * Fixing an issue with builds on Windows. * Trying another thing to fix the Windows build. * Fixed a minor bug in 'conduit::Node::set' that was causing memory problems when setting an existing object/list node to a scalar type. * Refactored the 'nestset' example code to use explicit transform operator to simplify code. * Add blueprint verify for structured version of adjsets * Small fixes in adjset verify * Added "windows" to adjset verify. * Change adjsets verify to allow for neighbor integers only * Add an example to verify the uniform version of adjset * Add manual build-up of adjest windows for testing * Fix adjset example to set coords for vertices, not zones * Add ratio for adjacency neighbors on different AMR levels * Add ratios to structured adjset blueprint mesh example Co-authored-by: Joseph Ciurej --- src/libs/blueprint/conduit_blueprint_mesh.cpp | 60 +- .../conduit_blueprint_mesh_examples.cpp | 571 ++++++++++++++++++ .../conduit_blueprint_mesh_examples.hpp | 3 + .../blueprint/t_blueprint_mesh_verify.cpp | 33 +- 4 files changed, 664 insertions(+), 3 deletions(-) diff --git a/src/libs/blueprint/conduit_blueprint_mesh.cpp b/src/libs/blueprint/conduit_blueprint_mesh.cpp index fce8f256c..f7ff1249d 100644 --- a/src/libs/blueprint/conduit_blueprint_mesh.cpp +++ b/src/libs/blueprint/conduit_blueprint_mesh.cpp @@ -4114,8 +4114,66 @@ mesh::adjset::verify(const Node &adjset, bool group_res = true; group_res &= verify_integer_field(protocol, chld, chld_info, "neighbors"); - group_res &= verify_integer_field(protocol, chld, chld_info, "values"); + if(chld.has_child("values")) + { + group_res &= verify_integer_field(protocol, chld, + chld_info, "values"); + } + else if(chld.has_child("windows")) + { + + group_res &= verify_object_field(protocol, chld, + chld_info, "windows"); + + bool windows_res = true; + NodeConstIterator witr = chld["windows"].children(); + while(witr.has_next()) + { + const Node &wndw = witr.next(); + const std::string wndw_name = witr.name(); + Node &wndw_info = chld_info["windows"][wndw_name]; + + bool window_res = true; + window_res &= verify_field_exists(protocol, wndw, + wndw_info, "origin") && + mesh::logical_dims::verify(wndw["origin"], + wndw_info["origin"]); + window_res &= verify_field_exists(protocol, wndw, + wndw_info, "dims") && + mesh::logical_dims::verify(wndw["dims"], + wndw_info["dims"]); + window_res &= verify_field_exists(protocol, wndw, + wndw_info, "ratio") && + mesh::logical_dims::verify(wndw["ratio"], + wndw_info["ratio"]); + + // verify that dimensions for "origin" and + // "dims" and "ratio" are the same + if(window_res) + { + index_t window_dim = wndw["origin"].number_of_children(); + window_res &= !wndw.has_child("dims") || + verify_object_field(protocol, wndw, + wndw_info, "dims", false, window_dim); + window_res &= !wndw.has_child("ratio") || + verify_object_field(protocol, wndw, + wndw_info, "ratio", false, window_dim); + } + + log::validation(wndw_info,window_res); + windows_res &= window_res; + } + + log::validation(chld_info["windows"],windows_res); + res &= windows_res; + if(chld.has_child("orientation")) + { + group_res &= verify_integer_field(protocol, chld, + chld_info, "orientation"); + } + } + log::validation(chld_info,group_res); groups_res &= group_res; } diff --git a/src/libs/blueprint/conduit_blueprint_mesh_examples.cpp b/src/libs/blueprint/conduit_blueprint_mesh_examples.cpp index cb95defd1..ee13827a1 100644 --- a/src/libs/blueprint/conduit_blueprint_mesh_examples.cpp +++ b/src/libs/blueprint/conduit_blueprint_mesh_examples.cpp @@ -2857,6 +2857,577 @@ misc(const std::string &mesh_type, } //----------------------------------------------------------------------------- +void +adjset_uniform(Node &res) +{ + for(index_t i = 0; i < 8; i++) + { + std::ostringstream oss; + oss << "domain" << i; + const std::string domain_name = oss.str(); + + Node &domain_node = res[domain_name]; + + domain_node["state/domain_id"].set(i); + Node &domain_coords = domain_node["coordsets/coords"]; + domain_coords["type"].set_string("uniform"); + domain_coords["dims/i"].set_int32(21); + domain_coords["dims/j"].set_int32(21); + domain_coords["spacing/dx"].set_float64(0.0125); + domain_coords["spacing/dy"].set_float64(0.025); + domain_coords["origin/x"].set_float64(0.25*(i/2)); + domain_coords["origin/y"].set_float64(0.5*(i%2)); + + Node &domain_topo = domain_node["topologies/topo"]; + domain_topo["elements/origin/i0"].set_int32(20*(i/2)); + domain_topo["elements/origin/j0"].set_int32(20*(i%2)); + domain_topo["type"].set_string("uniform"); + domain_topo["coordset"].set_string("coords"); + + Node &domain_adjsets = domain_node["adjsets/adjset"]; + domain_adjsets["association"].set_string("vertex"); + domain_adjsets["topology"].set_string("topo"); + Node &adjset_groups = domain_adjsets["groups"]; + + if (i == 0) + { + adjset_groups["group_000001/neighbors"] = DataType::int32(2); + adjset_groups["group_000002/neighbors"] = DataType::int32(2); + adjset_groups["group_000003/neighbors"] = DataType::int32(2); + adjset_groups["group_000001/neighbors"].as_int_array()[0] = 0; + adjset_groups["group_000002/neighbors"].as_int_array()[0] = 0; + adjset_groups["group_000003/neighbors"].as_int_array()[0] = 0; + adjset_groups["group_000001/neighbors"].as_int_array()[1] = 1; + adjset_groups["group_000002/neighbors"].as_int_array()[1] = 2; + adjset_groups["group_000003/neighbors"].as_int_array()[1] = 3; + Node &windows_node1 = adjset_groups["group_000001/windows"]; + for(index_t w = 0; w <= 1; w++) + { + std::ostringstream w_oss; + w_oss << "window_00000" << w; + const std::string window_name = w_oss.str(); + windows_node1[window_name]["origin/i"] = 0; + windows_node1[window_name]["origin/j"] = 20; + windows_node1[window_name]["dims/i"] = 21; + windows_node1[window_name]["dims/j"] = 1; + windows_node1[window_name]["ratio/i"] = 1; + windows_node1[window_name]["ratio/j"] = 1; + } + Node &windows_node2 = adjset_groups["group_000002/windows"]; + for(index_t w = 0; w <= 2; w += 2) + { + std::ostringstream w_oss; + w_oss << "window_00000" << w; + const std::string window_name = w_oss.str(); + windows_node2[window_name]["origin/i"] = 0; + windows_node2[window_name]["origin/j"] = 20; + windows_node2[window_name]["dims/i"] = 21; + windows_node2[window_name]["dims/j"] = 1; + windows_node2[window_name]["ratio/i"] = 1; + windows_node2[window_name]["ratio/j"] = 1; + } + Node &windows_node3 = adjset_groups["group_000003/windows"]; + for(index_t w = 0; w <= 3; w += 3) + { + std::ostringstream w_oss; + w_oss << "window_00000" << w; + const std::string window_name = w_oss.str(); + windows_node3[window_name]["origin/i"] = 20; + windows_node3[window_name]["origin/j"] = 20; + windows_node3[window_name]["dims/i"] = 1; + windows_node3[window_name]["dims/j"] = 1; + windows_node3[window_name]["ratio/i"] = 1; + windows_node3[window_name]["ratio/j"] = 1; + } + } + else if (i == 1) + { + adjset_groups["group_000000/neighbors"] = DataType::int32(2); + adjset_groups["group_000002/neighbors"] = DataType::int32(2); + adjset_groups["group_000003/neighbors"] = DataType::int32(2); + adjset_groups["group_000000/neighbors"].as_int_array()[0] = 1; + adjset_groups["group_000002/neighbors"].as_int_array()[0] = 1; + adjset_groups["group_000003/neighbors"].as_int_array()[0] = 1; + adjset_groups["group_000000/neighbors"].as_int_array()[1] = 0; + adjset_groups["group_000002/neighbors"].as_int_array()[1] = 2; + adjset_groups["group_000003/neighbors"].as_int_array()[1] = 3; + Node &windows_node0 = adjset_groups["group_000000/windows"]; + for(index_t w = 0; w <= 1; w++) + { + std::ostringstream w_oss; + w_oss << "window_00000" << w; + const std::string window_name = w_oss.str(); + windows_node0[window_name]["origin/i"] = 0; + windows_node0[window_name]["origin/j"] = 20; + windows_node0[window_name]["dims/i"] = 21; + windows_node0[window_name]["dims/j"] = 1; + windows_node0[window_name]["ratio/i"] = 1; + windows_node0[window_name]["ratio/j"] = 1; + } + Node &windows_node2 = adjset_groups["group_000002/windows"]; + for(index_t w = 1; w <= 2; w++) + { + std::ostringstream w_oss; + w_oss << "window_00000" << w; + const std::string window_name = w_oss.str(); + windows_node2[window_name]["origin/i"] = 20; + windows_node2[window_name]["origin/j"] = 20; + windows_node2[window_name]["dims/i"] = 1; + windows_node2[window_name]["dims/j"] = 1; + windows_node2[window_name]["ratio/i"] = 1; + windows_node2[window_name]["ratio/j"] = 1; + } + Node &windows_node3 = adjset_groups["group_000003/windows"]; + for(index_t w = 1; w <= 3; w += 2) + { + std::ostringstream w_oss; + w_oss << "window_00000" << w; + const std::string window_name = w_oss.str(); + windows_node3[window_name]["origin/i"] = 20; + windows_node3[window_name]["origin/j"] = 20; + windows_node3[window_name]["dims/i"] = 1; + windows_node3[window_name]["dims/j"] = 21; + windows_node3[window_name]["ratio/i"] = 1; + windows_node3[window_name]["ratio/j"] = 1; + } + } + else if (i == 2) + { + adjset_groups["group_000000/neighbors"] = DataType::int32(2); + adjset_groups["group_000001/neighbors"] = DataType::int32(2); + adjset_groups["group_000003/neighbors"] = DataType::int32(2); + adjset_groups["group_000004/neighbors"] = DataType::int32(2); + adjset_groups["group_000005/neighbors"] = DataType::int32(2); + adjset_groups["group_000000/neighbors"].as_int_array()[0] = 2; + adjset_groups["group_000001/neighbors"].as_int_array()[0] = 2; + adjset_groups["group_000003/neighbors"].as_int_array()[0] = 2; + adjset_groups["group_000004/neighbors"].as_int_array()[0] = 2; + adjset_groups["group_000005/neighbors"].as_int_array()[0] = 2; + adjset_groups["group_000000/neighbors"].as_int_array()[1] = 0; + adjset_groups["group_000001/neighbors"].as_int_array()[1] = 1; + adjset_groups["group_000003/neighbors"].as_int_array()[1] = 3; + adjset_groups["group_000004/neighbors"].as_int_array()[1] = 4; + adjset_groups["group_000005/neighbors"].as_int_array()[1] = 5; + Node &windows_node0 = adjset_groups["group_000000/windows"]; + for(index_t w = 0; w <= 2; w += 2) + { + std::ostringstream w_oss; + w_oss << "window_00000" << w; + const std::string window_name = w_oss.str(); + windows_node0[window_name]["origin/i"] = 20; + windows_node0[window_name]["origin/j"] = 0; + windows_node0[window_name]["dims/i"] = 1; + windows_node0[window_name]["dims/j"] = 21; + windows_node0[window_name]["ratio/i"] = 1; + windows_node0[window_name]["ratio/j"] = 1; + } + Node &windows_node1 = adjset_groups["group_000001/windows"]; + for(index_t w = 1; w <= 2; w++) + { + std::ostringstream w_oss; + w_oss << "window_00000" << w; + const std::string window_name = w_oss.str(); + windows_node1[window_name]["origin/i"] = 20; + windows_node1[window_name]["origin/j"] = 20; + windows_node1[window_name]["dims/i"] = 1; + windows_node1[window_name]["dims/j"] = 1; + windows_node1[window_name]["ratio/i"] = 1; + windows_node1[window_name]["ratio/j"] = 1; + } + Node &windows_node3 = adjset_groups["group_000003/windows"]; + for(index_t w = 2; w <= 3; w++) + { + std::ostringstream w_oss; + w_oss << "window_00000" << w; + const std::string window_name = w_oss.str(); + windows_node3[window_name]["origin/i"] = 20; + windows_node3[window_name]["origin/j"] = 20; + windows_node3[window_name]["dims/i"] = 21; + windows_node3[window_name]["dims/j"] = 1; + windows_node3[window_name]["ratio/i"] = 1; + windows_node3[window_name]["ratio/j"] = 1; + } + Node &windows_node4 = adjset_groups["group_000004/windows"]; + for(index_t w = 2; w <= 4; w += 2) + { + std::ostringstream w_oss; + w_oss << "window_00000" << w; + const std::string window_name = w_oss.str(); + windows_node4[window_name]["origin/i"] = 40; + windows_node4[window_name]["origin/j"] = 0; + windows_node4[window_name]["dims/i"] = 1; + windows_node4[window_name]["dims/j"] = 21; + windows_node4[window_name]["ratio/i"] = 1; + windows_node4[window_name]["ratio/j"] = 1; + } + Node &windows_node5 = adjset_groups["group_000005/windows"]; + for(index_t w = 2; w <= 5; w += 3) + { + std::ostringstream w_oss; + w_oss << "window_00000" << w; + const std::string window_name = w_oss.str(); + windows_node5[window_name]["origin/i"] = 40; + windows_node5[window_name]["origin/j"] = 20; + windows_node5[window_name]["dims/i"] = 1; + windows_node5[window_name]["dims/j"] = 1; + windows_node5[window_name]["ratio/i"] = 1; + windows_node5[window_name]["ratio/j"] = 1; + } + } + else if (i == 3) + { + adjset_groups["group_000000/neighbors"] = DataType::int32(2); + adjset_groups["group_000001/neighbors"] = DataType::int32(2); + adjset_groups["group_000002/neighbors"] = DataType::int32(2); + adjset_groups["group_000004/neighbors"] = DataType::int32(2); + adjset_groups["group_000005/neighbors"] = DataType::int32(2); + adjset_groups["group_000000/neighbors"].as_int_array()[0] = 3; + adjset_groups["group_000001/neighbors"].as_int_array()[0] = 3; + adjset_groups["group_000002/neighbors"].as_int_array()[0] = 3; + adjset_groups["group_000004/neighbors"].as_int_array()[0] = 3; + adjset_groups["group_000005/neighbors"].as_int_array()[0] = 3; + adjset_groups["group_000000/neighbors"].as_int_array()[1] = 0; + adjset_groups["group_000001/neighbors"].as_int_array()[1] = 1; + adjset_groups["group_000002/neighbors"].as_int_array()[1] = 2; + adjset_groups["group_000004/neighbors"].as_int_array()[1] = 4; + adjset_groups["group_000005/neighbors"].as_int_array()[1] = 5; + Node &windows_node0 = adjset_groups["group_000000/windows"]; + for(index_t w = 0; w <= 3; w += 3) + { + std::ostringstream w_oss; + w_oss << "window_00000" << w; + const std::string window_name = w_oss.str(); + windows_node0[window_name]["origin/i"] = 20; + windows_node0[window_name]["origin/j"] = 20; + windows_node0[window_name]["dims/i"] = 1; + windows_node0[window_name]["dims/j"] = 1; + windows_node0[window_name]["ratio/i"] = 1; + windows_node0[window_name]["ratio/j"] = 1; + } + Node &windows_node1 = adjset_groups["group_000001/windows"]; + for(index_t w = 1; w <= 3; w += 2) + { + std::ostringstream w_oss; + w_oss << "window_00000" << w; + const std::string window_name = w_oss.str(); + windows_node1[window_name]["origin/i"] = 20; + windows_node1[window_name]["origin/j"] = 20; + windows_node1[window_name]["dims/i"] = 1; + windows_node1[window_name]["dims/j"] = 21; + windows_node1[window_name]["ratio/i"] = 1; + windows_node1[window_name]["ratio/j"] = 1; + } + Node &windows_node2 = adjset_groups["group_000002/windows"]; + for(index_t w = 2; w <= 3; w++) + { + std::ostringstream w_oss; + w_oss << "window_00000" << w; + const std::string window_name = w_oss.str(); + windows_node2[window_name]["origin/i"] = 20; + windows_node2[window_name]["origin/j"] = 20; + windows_node2[window_name]["dims/i"] = 21; + windows_node2[window_name]["dims/j"] = 1; + windows_node2[window_name]["ratio/i"] = 1; + windows_node2[window_name]["ratio/j"] = 1; + } + Node &windows_node4 = adjset_groups["group_000004/windows"]; + for(index_t w = 3; w <= 4; w++) + { + std::ostringstream w_oss; + w_oss << "window_00000" << w; + const std::string window_name = w_oss.str(); + windows_node4[window_name]["origin/i"] = 40; + windows_node4[window_name]["origin/j"] = 20; + windows_node4[window_name]["dims/i"] = 1; + windows_node4[window_name]["dims/j"] = 1; + windows_node4[window_name]["ratio/i"] = 1; + windows_node4[window_name]["ratio/j"] = 1; + } + Node &windows_node5 = adjset_groups["group_000005/windows"]; + for(index_t w = 3; w <= 4; w += 2) + { + std::ostringstream w_oss; + w_oss << "window_00000" << w; + const std::string window_name = w_oss.str(); + windows_node5[window_name]["origin/i"] = 40; + windows_node5[window_name]["origin/j"] = 20; + windows_node5[window_name]["dims/i"] = 1; + windows_node5[window_name]["dims/j"] = 21; + windows_node5[window_name]["ratio/i"] = 1; + windows_node5[window_name]["ratio/j"] = 1; + } + } + else if (i == 4) + { + adjset_groups["group_000002/neighbors"] = DataType::int32(2); + adjset_groups["group_000003/neighbors"] = DataType::int32(2); + adjset_groups["group_000005/neighbors"] = DataType::int32(2); + adjset_groups["group_000006/neighbors"] = DataType::int32(2); + adjset_groups["group_000007/neighbors"] = DataType::int32(2); + adjset_groups["group_000002/neighbors"].as_int_array()[0] = 4; + adjset_groups["group_000003/neighbors"].as_int_array()[0] = 4; + adjset_groups["group_000005/neighbors"].as_int_array()[0] = 4; + adjset_groups["group_000006/neighbors"].as_int_array()[0] = 4; + adjset_groups["group_000007/neighbors"].as_int_array()[0] = 4; + adjset_groups["group_000002/neighbors"].as_int_array()[1] = 2; + adjset_groups["group_000003/neighbors"].as_int_array()[1] = 3; + adjset_groups["group_000005/neighbors"].as_int_array()[1] = 5; + adjset_groups["group_000006/neighbors"].as_int_array()[1] = 6; + adjset_groups["group_000007/neighbors"].as_int_array()[1] = 7; + Node &windows_node2 = adjset_groups["group_000002/windows"]; + for(index_t w = 2; w <= 4; w += 2) + { + std::ostringstream w_oss; + w_oss << "window_00000" << w; + const std::string window_name = w_oss.str(); + windows_node2[window_name]["origin/i"] = 40; + windows_node2[window_name]["origin/j"] = 0; + windows_node2[window_name]["dims/i"] = 1; + windows_node2[window_name]["dims/j"] = 21; + windows_node2[window_name]["ratio/i"] = 1; + windows_node2[window_name]["ratio/j"] = 1; + } + Node &windows_node3 = adjset_groups["group_000003/windows"]; + for(index_t w = 3; w <= 4; w++) + { + std::ostringstream w_oss; + w_oss << "window_00000" << w; + const std::string window_name = w_oss.str(); + windows_node3[window_name]["origin/i"] = 40; + windows_node3[window_name]["origin/j"] = 20; + windows_node3[window_name]["dims/i"] = 1; + windows_node3[window_name]["dims/j"] = 1; + windows_node3[window_name]["ratio/i"] = 1; + windows_node3[window_name]["ratio/j"] = 1; + } + Node &windows_node5 = adjset_groups["group_000005/windows"]; + for(index_t w = 4; w <= 5; w++) + { + std::ostringstream w_oss; + w_oss << "window_00000" << w; + const std::string window_name = w_oss.str(); + windows_node5[window_name]["origin/i"] = 40; + windows_node5[window_name]["origin/j"] = 20; + windows_node5[window_name]["dims/i"] = 21; + windows_node5[window_name]["dims/j"] = 1; + windows_node5[window_name]["ratio/i"] = 1; + windows_node5[window_name]["ratio/j"] = 1; + } + Node &windows_node6 = adjset_groups["group_000006/windows"]; + for(index_t w = 4; w <= 6; w += 2) + { + std::ostringstream w_oss; + w_oss << "window_00000" << w; + const std::string window_name = w_oss.str(); + windows_node6[window_name]["origin/i"] = 60; + windows_node6[window_name]["origin/j"] = 0; + windows_node6[window_name]["dims/i"] = 1; + windows_node6[window_name]["dims/j"] = 21; + windows_node6[window_name]["ratio/i"] = 1; + windows_node6[window_name]["ratio/j"] = 1; + } + Node &windows_node7 = adjset_groups["group_000007/windows"]; + for(index_t w = 4; w <= 7; w += 3) + { + std::ostringstream w_oss; + w_oss << "window_00000" << w; + const std::string window_name = w_oss.str(); + windows_node7[window_name]["origin/i"] = 60; + windows_node7[window_name]["origin/j"] = 20; + windows_node7[window_name]["dims/i"] = 1; + windows_node7[window_name]["dims/j"] = 1; + windows_node7[window_name]["ratio/i"] = 1; + windows_node7[window_name]["ratio/j"] = 1; + } + } + else if (i == 5) + { + adjset_groups["group_000002/neighbors"] = DataType::int32(2); + adjset_groups["group_000003/neighbors"] = DataType::int32(2); + adjset_groups["group_000004/neighbors"] = DataType::int32(2); + adjset_groups["group_000006/neighbors"] = DataType::int32(2); + adjset_groups["group_000007/neighbors"] = DataType::int32(2); + adjset_groups["group_000002/neighbors"].as_int_array()[0] = 5; + adjset_groups["group_000003/neighbors"].as_int_array()[0] = 5; + adjset_groups["group_000004/neighbors"].as_int_array()[0] = 5; + adjset_groups["group_000006/neighbors"].as_int_array()[0] = 5; + adjset_groups["group_000007/neighbors"].as_int_array()[0] = 5; + adjset_groups["group_000002/neighbors"].as_int_array()[1] = 2; + adjset_groups["group_000003/neighbors"].as_int_array()[1] = 3; + adjset_groups["group_000004/neighbors"].as_int_array()[1] = 4; + adjset_groups["group_000006/neighbors"].as_int_array()[1] = 6; + adjset_groups["group_000007/neighbors"].as_int_array()[1] = 7; + Node &windows_node2 = adjset_groups["group_000002/windows"]; + for(index_t w = 2; w <= 5; w += 3) + { + std::ostringstream w_oss; + w_oss << "window_00000" << w; + const std::string window_name = w_oss.str(); + windows_node2[window_name]["origin/i"] = 40; + windows_node2[window_name]["origin/j"] = 20; + windows_node2[window_name]["dims/i"] = 1; + windows_node2[window_name]["dims/j"] = 1; + windows_node2[window_name]["ratio/i"] = 1; + windows_node2[window_name]["ratio/j"] = 1; + } + Node &windows_node3 = adjset_groups["group_000003/windows"]; + for(index_t w = 3; w <= 5; w += 2) + { + std::ostringstream w_oss; + w_oss << "window_00000" << w; + const std::string window_name = w_oss.str(); + windows_node3[window_name]["origin/i"] = 40; + windows_node3[window_name]["origin/j"] = 20; + windows_node3[window_name]["dims/i"] = 1; + windows_node3[window_name]["dims/j"] = 21; + windows_node3[window_name]["ratio/i"] = 1; + windows_node3[window_name]["ratio/j"] = 1; + } + Node &windows_node4 = adjset_groups["group_000004/windows"]; + for(index_t w = 4; w <= 5; w++) + { + std::ostringstream w_oss; + w_oss << "window_00000" << w; + const std::string window_name = w_oss.str(); + windows_node4[window_name]["origin/i"] = 40; + windows_node4[window_name]["origin/j"] = 20; + windows_node4[window_name]["dims/i"] = 1; + windows_node4[window_name]["dims/j"] = 21; + windows_node4[window_name]["ratio/i"] = 1; + windows_node4[window_name]["ratio/j"] = 1; + } + Node &windows_node6 = adjset_groups["group_000006/windows"]; + for(index_t w = 5; w <= 6; w++) + { + std::ostringstream w_oss; + w_oss << "window_00000" << w; + const std::string window_name = w_oss.str(); + windows_node6[window_name]["origin/i"] = 60; + windows_node6[window_name]["origin/j"] = 20; + windows_node6[window_name]["dims/i"] = 1; + windows_node6[window_name]["dims/j"] = 1; + windows_node6[window_name]["ratio/i"] = 1; + windows_node6[window_name]["ratio/j"] = 1; + } + Node &windows_node7 = adjset_groups["group_000007/windows"]; + for(index_t w = 5; w <= 7; w++) + { + std::ostringstream w_oss; + w_oss << "window_00000" << w; + const std::string window_name = w_oss.str(); + windows_node7[window_name]["origin/i"] = 60; + windows_node7[window_name]["origin/j"] = 20; + windows_node7[window_name]["dims/i"] = 1; + windows_node7[window_name]["dims/j"] = 21; + windows_node7[window_name]["ratio/i"] = 1; + windows_node7[window_name]["ratio/j"] = 1; + } + } + else if (i == 6) + { + adjset_groups["group_000004/neighbors"] = DataType::int32(2); + adjset_groups["group_000005/neighbors"] = DataType::int32(2); + adjset_groups["group_000007/neighbors"] = DataType::int32(2); + adjset_groups["group_000004/neighbors"].as_int_array()[0] = 6; + adjset_groups["group_000005/neighbors"].as_int_array()[0] = 6; + adjset_groups["group_000007/neighbors"].as_int_array()[0] = 6; + adjset_groups["group_000004/neighbors"].as_int_array()[1] = 4; + adjset_groups["group_000005/neighbors"].as_int_array()[1] = 5; + adjset_groups["group_000007/neighbors"].as_int_array()[1] = 7; + Node &windows_node4 = adjset_groups["group_000004/windows"]; + for(index_t w = 4; w <= 6; w += 2) + { + std::ostringstream w_oss; + w_oss << "window_00000" << w; + const std::string window_name = w_oss.str(); + windows_node4[window_name]["origin/i"] = 60; + windows_node4[window_name]["origin/j"] = 0; + windows_node4[window_name]["dims/i"] = 1; + windows_node4[window_name]["dims/j"] = 21; + windows_node4[window_name]["ratio/i"] = 1; + windows_node4[window_name]["ratio/j"] = 1; + } + Node &windows_node5 = adjset_groups["group_000005/windows"]; + for(index_t w = 5; w <= 6; w++) + { + std::ostringstream w_oss; + w_oss << "window_00000" << w; + const std::string window_name = w_oss.str(); + windows_node5[window_name]["origin/i"] = 60; + windows_node5[window_name]["origin/j"] = 20; + windows_node5[window_name]["dims/i"] = 1; + windows_node5[window_name]["dims/j"] = 1; + windows_node5[window_name]["ratio/i"] = 1; + windows_node5[window_name]["ratio/j"] = 1; + } + Node &windows_node7 = adjset_groups["group_000007/windows"]; + for(index_t w = 6; w <= 7; w++) + { + std::ostringstream w_oss; + w_oss << "window_00000" << w; + const std::string window_name = w_oss.str(); + windows_node7[window_name]["origin/i"] = 60; + windows_node7[window_name]["origin/j"] = 20; + windows_node7[window_name]["dims/i"] = 21; + windows_node7[window_name]["dims/j"] = 1; + windows_node7[window_name]["ratio/i"] = 1; + windows_node7[window_name]["ratio/j"] = 1; + } + } + else if (i == 7) + { + adjset_groups["group_000004/neighbors"] = DataType::int32(2); + adjset_groups["group_000005/neighbors"] = DataType::int32(2); + adjset_groups["group_000006/neighbors"] = DataType::int32(2); + adjset_groups["group_000004/neighbors"].as_int_array()[0] = 7; + adjset_groups["group_000005/neighbors"].as_int_array()[0] = 7; + adjset_groups["group_000006/neighbors"].as_int_array()[0] = 7; + adjset_groups["group_000004/neighbors"].as_int_array()[1] = 4; + adjset_groups["group_000005/neighbors"].as_int_array()[1] = 5; + adjset_groups["group_000006/neighbors"].as_int_array()[1] = 6; + Node &windows_node4 = adjset_groups["group_000004/windows"]; + for(index_t w = 4; w <= 7; w += 3) + { + std::ostringstream w_oss; + w_oss << "window_00000" << w; + const std::string window_name = w_oss.str(); + windows_node4[window_name]["origin/i"] = 60; + windows_node4[window_name]["origin/j"] = 20; + windows_node4[window_name]["dims/i"] = 1; + windows_node4[window_name]["dims/j"] = 1; + windows_node4[window_name]["ratio/i"] = 1; + windows_node4[window_name]["ratio/j"] = 1; + } + Node &windows_node5 = adjset_groups["group_000005/windows"]; + for(index_t w = 5; w <= 7; w += 2) + { + std::ostringstream w_oss; + w_oss << "window_00000" << w; + const std::string window_name = w_oss.str(); + windows_node5[window_name]["origin/i"] = 60; + windows_node5[window_name]["origin/j"] = 20; + windows_node5[window_name]["dims/i"] = 1; + windows_node5[window_name]["dims/j"] = 21; + windows_node5[window_name]["ratio/i"] = 1; + windows_node5[window_name]["ratio/j"] = 1; + } + Node &windows_node6 = adjset_groups["group_000006/windows"]; + for(index_t w = 6; w <= 7; w++) + { + std::ostringstream w_oss; + w_oss << "window_00000" << w; + const std::string window_name = w_oss.str(); + windows_node6[window_name]["origin/i"] = 60; + windows_node6[window_name]["origin/j"] = 20; + windows_node6[window_name]["dims/i"] = 21; + windows_node6[window_name]["dims/j"] = 1; + windows_node6[window_name]["ratio/i"] = 1; + windows_node6[window_name]["ratio/j"] = 1; + } + } + } +} + } //----------------------------------------------------------------------------- // -- end conduit::blueprint::mesh::examples -- diff --git a/src/libs/blueprint/conduit_blueprint_mesh_examples.hpp b/src/libs/blueprint/conduit_blueprint_mesh_examples.hpp index d12a43be4..a89ff2430 100644 --- a/src/libs/blueprint/conduit_blueprint_mesh_examples.hpp +++ b/src/libs/blueprint/conduit_blueprint_mesh_examples.hpp @@ -129,6 +129,9 @@ namespace examples conduit::index_t ny, conduit::index_t nz, conduit::Node &res); + + /// Generates a mesh that uses uniform adjsets + void CONDUIT_BLUEPRINT_API adjset_uniform(conduit::Node &res); } //----------------------------------------------------------------------------- // -- end conduit::blueprint::mesh::examples -- diff --git a/src/tests/blueprint/t_blueprint_mesh_verify.cpp b/src/tests/blueprint/t_blueprint_mesh_verify.cpp index 957411ac8..b1cb5d54a 100644 --- a/src/tests/blueprint/t_blueprint_mesh_verify.cpp +++ b/src/tests/blueprint/t_blueprint_mesh_verify.cpp @@ -1122,7 +1122,7 @@ TEST(conduit_blueprint_mesh_verify, adjset_general) n["groups"].reset(); n["groups"]["g1"]["neighbors"].set(DataType::int32(5)); - CHECK_MESH(verify_adjset,n,info,false); + CHECK_MESH(verify_adjset,n,info,true); n["groups"]["g1"]["values"].set(DataType::float32(5)); CHECK_MESH(verify_adjset,n,info,false); n["groups"]["g1"]["values"].set(DataType::int32(5)); @@ -1153,6 +1153,35 @@ TEST(conduit_blueprint_mesh_verify, adjset_general) } } + +//----------------------------------------------------------------------------- +TEST(conduit_blueprint_mesh_verify, adjset_structured) +{ + VerifyFun verify_adjset_funs[] = { + blueprint::mesh::adjset::verify, + verify_adjset_protocol}; + + for(index_t fi = 0; fi < 2; fi++) + { + VerifyFun verify_adjset = verify_adjset_funs[fi]; + + Node mesh, info; + CHECK_MESH(verify_adjset,mesh,info,false); + + blueprint::mesh::examples::adjset_uniform(mesh); + + NodeConstIterator itr = mesh.children(); + while(itr.has_next()) + { + const Node &chld= itr.next(); + const std::string chld_name = itr.name(); + + const Node& n = chld["adjsets/adjset"]; + CHECK_MESH(verify_adjset,n,info[chld_name],true); + } + } +} + /// Mesh Domain Nesting (AMR) Tests /// //----------------------------------------------------------------------------- @@ -2227,7 +2256,7 @@ TEST(conduit_blueprint_mesh_verify, mesh_general) Node &groups = domain["adjsets"]["mesh"]["groups"]; groups["g1"]["neighbors"].set(DataType::int32(10)); - CHECK_MESH(verify_mesh,mesh,info,false); + CHECK_MESH(verify_mesh,mesh,info,true); groups["g1"]["values"].set(DataType::float32(10)); CHECK_MESH(verify_mesh,mesh,info,false); groups["g1"]["values"].set(DataType::int32(10)); From 017983e5e220a8ec625c30551a5e54a05ecf3e37 Mon Sep 17 00:00:00 2001 From: Cyrus Harrison Date: Mon, 20 Apr 2020 16:41:19 -0700 Subject: [PATCH 3/5] add output of adjset_uniform example to hdf5 (#547) * add output of adjset_uniform example to hdf5 * update changelog --- CHANGELOG.md | 4 +++ .../conduit_blueprint_mesh_examples.cpp | 14 +++++++- .../blueprint/t_blueprint_mesh_examples.cpp | 33 +++++++++++++++++++ 3 files changed, 50 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 292799f4c..4b264e97a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,10 @@ and this project aspires to adhere to [Semantic Versioning](https://semver.org/s - Added an open mode option to RelayIOHandle. See RelayIOHandle docs (https://llnl-conduit.readthedocs.io/en/latest/relay_io.html#relay-i-o-handle-interface) for more details. - Added the conduit.relay.mpi Python module to support Relay MPI in Python. +#### Blueprint +- Added support for Adjacency sets for Structured Mesh Topologies. See the `blueprint::mesh::examples::adjset_uniform` example. + + ### Fixed #### General diff --git a/src/libs/blueprint/conduit_blueprint_mesh_examples.cpp b/src/libs/blueprint/conduit_blueprint_mesh_examples.cpp index ee13827a1..1f2992130 100644 --- a/src/libs/blueprint/conduit_blueprint_mesh_examples.cpp +++ b/src/libs/blueprint/conduit_blueprint_mesh_examples.cpp @@ -2863,7 +2863,7 @@ adjset_uniform(Node &res) for(index_t i = 0; i < 8; i++) { std::ostringstream oss; - oss << "domain" << i; + oss << "domain_" << std::setfill('0') << std::setw(6) << i; const std::string domain_name = oss.str(); Node &domain_node = res[domain_name]; @@ -2884,6 +2884,18 @@ adjset_uniform(Node &res) domain_topo["type"].set_string("uniform"); domain_topo["coordset"].set_string("coords"); + // add a simple field that has the domain id + Node &domain_field = domain_node["fields/id"]; + domain_field["association"] = "element"; + domain_field["topology"] = "topo"; + domain_field["values"] = DataType::int32(20 * 20); + + int32_array vals = domain_field["values"].value(); + for(int j=0;j<20*20;j++) + { + vals[j] = i; + } + Node &domain_adjsets = domain_node["adjsets/adjset"]; domain_adjsets["association"].set_string("vertex"); domain_adjsets["topology"].set_string("topo"); diff --git a/src/tests/blueprint/t_blueprint_mesh_examples.cpp b/src/tests/blueprint/t_blueprint_mesh_examples.cpp index 964eeb2f4..7ac71081c 100644 --- a/src/tests/blueprint/t_blueprint_mesh_examples.cpp +++ b/src/tests/blueprint/t_blueprint_mesh_examples.cpp @@ -504,6 +504,39 @@ TEST(conduit_blueprint_mesh_examples, check_gen_index_state_prop) EXPECT_TRUE(idx.has_path("state/time")); } +//----------------------------------------------------------------------------- +TEST(conduit_blueprint_mesh_examples, save_adjset_uniform) +{ + + Node io_protos; + relay::io::about(io_protos["io"]); + bool hdf5_enabled =io_protos["io/protocols/hdf5"].as_string() == "enabled"; + + // skip if we don't have hdf5 since this example has several domains + if(!hdf5_enabled) + return; + + Node mesh,idx; + blueprint::mesh::examples::adjset_uniform(mesh); + blueprint::mesh::generate_index(mesh[0], + "", + 8, + mesh["blueprint_index/adj_uniform"]); + + mesh["protocol/name"] = "hdf5"; + mesh["protocol/version"] = PROTOCOL_VER; + + mesh["number_of_files"] = 1; + mesh["number_of_trees"] = 8; + mesh["file_pattern"] = "adj_uniform_example.blueprint_root"; + mesh["tree_pattern"] = "domain_%06d"; + + CONDUIT_INFO("Creating: adj_uniform_example.blueprint_root") + relay::io::save(mesh,"adj_uniform_example.blueprint_root","json"); +} + + + //----------------------------------------------------------------------------- int main(int argc, char* argv[]) From 647bec039d80bd0194509442bd3ad2cd29632ff6 Mon Sep 17 00:00:00 2001 From: Cyrus Harrison Date: Thu, 23 Apr 2020 08:49:41 -0700 Subject: [PATCH 4/5] add to_string methods to core classes (#549) * py relay mpi remove uness debug print statements * add to_string methods to core classes * schema: deprecate detailed arg for save and to_json methods * eye crossing docs updates to use beg end sphinx style * tweak yaml output to supress extra newlines * py add node to_string() * update change log --- CHANGELOG.md | 7 + scripts/regen_docs_outputs.py | 37 +++ src/docs/sphinx/blueprint.rst | 14 +- src/docs/sphinx/blueprint_mesh.rst | 102 ++++--- src/docs/sphinx/relay_io.rst | 59 ++-- src/docs/sphinx/relay_mpi.rst | 22 +- .../t_conduit_docs_blueprint_demos_out.txt | 287 ++++++++++++++++++ .../t_conduit_docs_blueprint_examples_out.txt | 122 ++++---- ...onduit_docs_python_tutorial_basics_out.txt | 162 ---------- ..._conduit_docs_python_tutorial_json_out.txt | 51 ---- ...conduit_docs_python_tutorial_parse_out.txt | 76 ----- ..._tutorial_relay_io_handle_examples_out.txt | 74 ----- ...uit_docs_relay_io_generic_examples_out.txt | 236 +++++--------- ...duit_docs_relay_io_handle_examples_out.txt | 50 ++- ...onduit_docs_relay_io_hdf5_examples_out.txt | 140 ++++----- .../t_conduit_docs_tutorial_basics_out.txt | 179 +++++------ .../t_conduit_docs_tutorial_errors_out.txt | 26 +- .../t_conduit_docs_tutorial_numeric_out.txt | 58 ++-- .../t_conduit_docs_tutorial_ownership_out.txt | 63 ++-- .../t_conduit_docs_tutorial_parse_out.txt | 155 ++++------ ...onduit_docs_tutorial_python_basics_out.txt | 123 ++++++++ ...nduit_docs_tutorial_python_numeric_out.txt | 35 +++ ...uit_docs_tutorial_python_ownership_out.txt | 57 ++++ ...conduit_docs_tutorial_python_parse_out.txt | 73 +++++ ...al_python_relay_io_handle_examples_out.txt | 64 ++++ ...utorial_python_relay_mpi_examples_out.txt} | 14 +- src/docs/sphinx/tutorial_cpp_basics.rst | 44 ++- src/docs/sphinx/tutorial_cpp_errors.rst | 15 +- src/docs/sphinx/tutorial_cpp_json.rst | 192 ------------ src/docs/sphinx/tutorial_cpp_numeric.rst | 48 +-- src/docs/sphinx/tutorial_cpp_ownership.rst | 6 +- src/docs/sphinx/tutorial_python_basics.rst | 56 ++-- src/docs/sphinx/tutorial_python_parse.rst | 76 ++--- src/libs/conduit/conduit_data_array.cpp | 45 ++- src/libs/conduit/conduit_data_array.hpp | 14 + src/libs/conduit/conduit_data_type.cpp | 32 ++ src/libs/conduit/conduit_data_type.hpp | 11 +- src/libs/conduit/conduit_node.cpp | 150 +++++++-- src/libs/conduit/conduit_node.hpp | 32 ++ src/libs/conduit/conduit_schema.cpp | 145 +++++++-- src/libs/conduit/conduit_schema.hpp | 76 ++++- src/libs/conduit/python/conduit_python.cpp | 110 ++++++- .../relay/python/conduit_relay_mpi_python.cpp | 3 - .../conduit/python/t_python_conduit_node.py | 64 ++-- src/tests/conduit/t_conduit_array.cpp | 22 +- .../conduit/t_conduit_datatype_tests.cpp | 2 +- src/tests/conduit/t_conduit_node.cpp | 284 ++++++++++------- src/tests/conduit/t_conduit_schema.cpp | 52 ++++ src/tests/conduit/t_conduit_yaml.cpp | 18 ++ .../docs/t_conduit_docs_blueprint_demos.cpp | 23 +- .../t_conduit_docs_blueprint_examples.cpp | 19 +- ...conduit_docs_relay_io_generic_examples.cpp | 64 ++-- ..._conduit_docs_relay_io_handle_examples.cpp | 7 +- .../t_conduit_docs_relay_io_hdf5_examples.cpp | 22 +- .../docs/t_conduit_docs_tutorial_basics.cpp | 57 ++-- .../docs/t_conduit_docs_tutorial_errors.cpp | 23 +- .../docs/t_conduit_docs_tutorial_numeric.cpp | 50 +-- .../t_conduit_docs_tutorial_ownership.cpp | 7 +- .../docs/t_conduit_docs_tutorial_parse.cpp | 18 +- .../t_conduit_docs_tutorial_python_basics.py | 56 ++-- .../t_conduit_docs_tutorial_python_numeric.py | 13 +- ..._conduit_docs_tutorial_python_ownership.py | 14 +- .../t_conduit_docs_tutorial_python_parse.py | 24 +- ...utorial_python_relay_io_handle_examples.py | 11 +- ...docs_tutorial_python_relay_mpi_examples.py | 1 - 65 files changed, 2356 insertions(+), 1836 deletions(-) create mode 100755 scripts/regen_docs_outputs.py create mode 100644 src/docs/sphinx/t_conduit_docs_blueprint_demos_out.txt delete mode 100644 src/docs/sphinx/t_conduit_docs_python_tutorial_basics_out.txt delete mode 100644 src/docs/sphinx/t_conduit_docs_python_tutorial_json_out.txt delete mode 100644 src/docs/sphinx/t_conduit_docs_python_tutorial_parse_out.txt delete mode 100644 src/docs/sphinx/t_conduit_docs_python_tutorial_relay_io_handle_examples_out.txt create mode 100644 src/docs/sphinx/t_conduit_docs_tutorial_python_basics_out.txt create mode 100644 src/docs/sphinx/t_conduit_docs_tutorial_python_numeric_out.txt create mode 100644 src/docs/sphinx/t_conduit_docs_tutorial_python_ownership_out.txt create mode 100644 src/docs/sphinx/t_conduit_docs_tutorial_python_parse_out.txt create mode 100644 src/docs/sphinx/t_conduit_docs_tutorial_python_relay_io_handle_examples_out.txt rename src/docs/sphinx/{t_conduit_docs_python_tutorial_relay_mpi_examples_out.txt => t_conduit_docs_tutorial_python_relay_mpi_examples_out.txt} (96%) delete mode 100644 src/docs/sphinx/tutorial_cpp_json.rst diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b264e97a..fdd725f47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ and this project aspires to adhere to [Semantic Versioning](https://semver.org/s - Added Node::fetch_existing and Schema::fetch_existing methods, which provide access to existing paths or error when given a bad path. - Added Node::add_child() and Node::remove_child() to support direct operatrions and cases where names have `/`s. - Added a set of conduit::utils::log::remove_* filtering functions, which process conduit log/info nodes and strip out the requested information (useful for focusing the often verbose output in log/info nodes). +- Added to_string() and to_string_default() methods to Node, Schema, DataType, and DataArray. These methods alias either to_yaml() or to_json(). Long term yaml will be preferred over json, but Schema does not support yaml yet. +- Added helper script (scripts/regen_docs_outputs.py) that regenerates all example outputs used Conduit's Sphinx docs. #### Relay - Added an open mode option to RelayIOHandle. See RelayIOHandle docs (https://llnl-conduit.readthedocs.io/en/latest/relay_io.html#relay-i-o-handle-interface) for more details. @@ -36,6 +38,11 @@ and this project aspires to adhere to [Semantic Versioning](https://semver.org/s #### General - Node::fetch_child and Schema::fetch_child are deprecated in favor of the more clearly named Node::fetch_existing and Schema::fetch_existing. fetch_child variants still exist, but will be removed in a future release. +- Python str() methods for Node, Schema, and DataType now use their new to_string() methods. +- DataArray::to_json(std::ostring &) is deprecated in favor DataArray::to_json_stream. to_json(std::ostring &) will be removed in a future release. +- Schema::to_json and Schema::save variants with detailed (bool) arg are deprecated. The detailed arg was never used. These methods will be removed in a future release. +- Node::print() now prints yaml instead of json. +- Sphinx Docs code examples and outputs are now included using start-after and end-before style includes. #### Relay - Provide more context when a Conduit Node cannot be written to a HDF5 file because it is incompatible with the existing HDF5 tree. Error messages now provide the full path and details about the incompatibility. diff --git a/scripts/regen_docs_outputs.py b/scripts/regen_docs_outputs.py new file mode 100755 index 000000000..f1e3f948a --- /dev/null +++ b/scripts/regen_docs_outputs.py @@ -0,0 +1,37 @@ +#! /usr/bin/env python +# note: run in build dir, expects src to be ../src from build dir +import subprocess +import os + +from os.path import join as pjoin + +tests = ["t_conduit_docs_tutorial_basics", + "t_conduit_docs_tutorial_numeric", + "t_conduit_docs_tutorial_parse", + "t_conduit_docs_tutorial_ownership", + "t_conduit_docs_tutorial_errors", + # bp + "t_conduit_docs_blueprint_demos", + "t_conduit_docs_blueprint_examples", + "t_conduit_docs_relay_io_generic_examples", + "t_conduit_docs_relay_io_handle_examples", + "t_conduit_docs_relay_io_hdf5_examples", + # python + "t_conduit_docs_tutorial_python_basics", + "t_conduit_docs_tutorial_python_numeric", + "t_conduit_docs_tutorial_python_ownership", + "t_conduit_docs_tutorial_python_parse", + "t_conduit_docs_tutorial_python_relay_io_handle_examples", + "t_conduit_docs_tutorial_python_relay_mpi_examples"] + + +dest_dir = "../src/docs/sphinx/" +out_suffix = "_out.txt" + +for test in tests: + if "python" in test: + cmd = "ctest -VV -R {} > {}_out.txt".format(test,pjoin(dest_dir,test)) + else: + cmd = "tests/docs/{} > {}_out.txt".format(test,pjoin(dest_dir,test)) + print(cmd) + subprocess.call(cmd,shell=True) diff --git a/src/docs/sphinx/blueprint.rst b/src/docs/sphinx/blueprint.rst index e419a0ad3..0bccff90b 100644 --- a/src/docs/sphinx/blueprint.rst +++ b/src/docs/sphinx/blueprint.rst @@ -86,12 +86,15 @@ Blueprint provides a generic top level ``verify()`` method, which exposes the ve ``verify()`` returns true if the passed Node *node* conforms to the named protocol. It also provides details about the verification, including specific errors in the passed *info* Node. .. literalinclude:: ../../tests/docs/t_conduit_docs_blueprint_examples.cpp - :lines: 65-79 + :start-after: BEGIN_EXAMPLE("blueprint_example_1") + :end-before: END_EXAMPLE("blueprint_example_1") :language: cpp :dedent: 4 + .. literalinclude:: t_conduit_docs_blueprint_examples_out.txt - :lines: 10-19 + :start-after: BEGIN_EXAMPLE("blueprint_example_1") + :end-before: END_EXAMPLE("blueprint_example_1") @@ -99,13 +102,14 @@ Methods for specific protocols are grouped in namespaces: .. literalinclude:: ../../tests/docs/t_conduit_docs_blueprint_examples.cpp - :lines: 90-116 + :start-after: BEGIN_EXAMPLE("blueprint_example_2") + :end-before: END_EXAMPLE("blueprint_example_2") :language: cpp :dedent: 4 .. literalinclude:: t_conduit_docs_blueprint_examples_out.txt - :lines: 26-83 - + :start-after: BEGIN_EXAMPLE("blueprint_example_2") + :end-before: END_EXAMPLE("blueprint_example_2") diff --git a/src/docs/sphinx/blueprint_mesh.rst b/src/docs/sphinx/blueprint_mesh.rst index 21580961d..2c57f1872 100644 --- a/src/docs/sphinx/blueprint_mesh.rst +++ b/src/docs/sphinx/blueprint_mesh.rst @@ -629,16 +629,17 @@ Uniform * **Usage Example** .. literalinclude:: ../../tests/docs/t_conduit_docs_blueprint_demos.cpp - :lines: 138-143 + :start-after: BEGIN_EXAMPLE("blueprint_demo_basic_uniform") + :end-before: END_EXAMPLE("blueprint_demo_basic_uniform") :language: cpp :dedent: 4 * **Result** -.. literalinclude:: ../../tests/docs/t_conduit_docs_blueprint_demos.cpp - :lines: 146-187 - :language: cpp - :dedent: 4 +.. literalinclude:: t_conduit_docs_blueprint_demos_out.txt + :start-after: BEGIN_EXAMPLE("blueprint_demo_basic_uniform") + :end-before: END_EXAMPLE("blueprint_demo_basic_uniform") + :language: yaml * **Visual** @@ -654,16 +655,17 @@ Rectilinear * **Usage Example** .. literalinclude:: ../../tests/docs/t_conduit_docs_blueprint_demos.cpp - :lines: 197-201 + :start-after: BEGIN_EXAMPLE("blueprint_demo_basic_rectilinear") + :end-before: END_EXAMPLE("blueprint_demo_basic_rectilinear") :language: cpp :dedent: 4 * **Result** -.. literalinclude:: ../../tests/docs/t_conduit_docs_blueprint_demos.cpp - :lines: 204-235 - :language: cpp - :dedent: 4 +.. literalinclude:: t_conduit_docs_blueprint_demos_out.txt + :start-after: BEGIN_EXAMPLE("blueprint_demo_basic_rectilinear") + :end-before: END_EXAMPLE("blueprint_demo_basic_rectilinear") + :language: yaml * **Visual** @@ -679,16 +681,17 @@ Structured * **Usage Example** .. literalinclude:: ../../tests/docs/t_conduit_docs_blueprint_demos.cpp - :lines: 244-249 + :start-after: BEGIN_EXAMPLE("blueprint_demo_basic_structured") + :end-before: END_EXAMPLE("blueprint_demo_basic_structured") :language: cpp :dedent: 4 * **Result** -.. literalinclude:: ../../tests/docs/t_conduit_docs_blueprint_demos.cpp - :lines: 252-291 - :language: cpp - :dedent: 4 +.. literalinclude:: t_conduit_docs_blueprint_demos_out.txt + :start-after: BEGIN_EXAMPLE("blueprint_demo_basic_structured") + :end-before: END_EXAMPLE("blueprint_demo_basic_structured") + :language: yaml * **Visual** @@ -704,16 +707,17 @@ Tris * **Usage Example** .. literalinclude:: ../../tests/docs/t_conduit_docs_blueprint_demos.cpp - :lines: 300-305 + :start-after: BEGIN_EXAMPLE("blueprint_demo_basic_tris") + :end-before: END_EXAMPLE("blueprint_demo_basic_tris") :language: cpp :dedent: 4 * **Result** -.. literalinclude:: ../../tests/docs/t_conduit_docs_blueprint_demos.cpp - :lines: 308-344 - :language: cpp - :dedent: 4 +.. literalinclude:: t_conduit_docs_blueprint_demos_out.txt + :start-after: BEGIN_EXAMPLE("blueprint_demo_basic_tris") + :end-before: END_EXAMPLE("blueprint_demo_basic_tris") + :language: yaml * **Visual** @@ -729,16 +733,17 @@ Quads * **Usage Example** .. literalinclude:: ../../tests/docs/t_conduit_docs_blueprint_demos.cpp - :lines: 353-358 + :start-after: BEGIN_EXAMPLE("blueprint_demo_basic_quads") + :end-before: END_EXAMPLE("blueprint_demo_basic_quads") :language: cpp :dedent: 4 * **Result** -.. literalinclude:: ../../tests/docs/t_conduit_docs_blueprint_demos.cpp - :lines: 361-397 - :language: cpp - :dedent: 4 +.. literalinclude:: t_conduit_docs_blueprint_demos_out.txt + :start-after: BEGIN_EXAMPLE("blueprint_demo_basic_quads") + :end-before: END_EXAMPLE("blueprint_demo_basic_quads") + :language: yaml * **Visual** @@ -754,16 +759,17 @@ Polygons * **Usage Example** .. literalinclude:: ../../tests/docs/t_conduit_docs_blueprint_demos.cpp - :lines: 514-519 + :start-after: BEGIN_EXAMPLE("blueprint_demo_basic_polygons") + :end-before: END_EXAMPLE("blueprint_demo_basic_polygons") :language: cpp :dedent: 4 * **Result** -.. literalinclude:: ../../tests/docs/t_conduit_docs_blueprint_demos.cpp - :lines: 522-558 - :language: cpp - :dedent: 4 +.. literalinclude:: t_conduit_docs_blueprint_demos_out.txt + :start-after: BEGIN_EXAMPLE("blueprint_demo_basic_polygons") + :end-before: END_EXAMPLE("blueprint_demo_basic_polygons") + :language: yaml * **Visual** @@ -779,16 +785,17 @@ Tets * **Usage Example** .. literalinclude:: ../../tests/docs/t_conduit_docs_blueprint_demos.cpp - :lines: 406-411 + :start-after: BEGIN_EXAMPLE("blueprint_demo_basic_tets") + :end-before: END_EXAMPLE("blueprint_demo_basic_tets") :language: cpp :dedent: 4 * **Result** -.. literalinclude:: ../../tests/docs/t_conduit_docs_blueprint_demos.cpp - :lines: 414-451 - :language: cpp - :dedent: 4 +.. literalinclude:: t_conduit_docs_blueprint_demos_out.txt + :start-after: BEGIN_EXAMPLE("blueprint_demo_basic_tets") + :end-before: END_EXAMPLE("blueprint_demo_basic_tets") + :language: yaml * **Visual** @@ -804,16 +811,17 @@ Hexs * **Usage Example** .. literalinclude:: ../../tests/docs/t_conduit_docs_blueprint_demos.cpp - :lines: 460-465 + :start-after: BEGIN_EXAMPLE("blueprint_demo_basic_hexs") + :end-before: END_EXAMPLE("blueprint_demo_basic_hexs") :language: cpp :dedent: 4 * **Result** -.. literalinclude:: ../../tests/docs/t_conduit_docs_blueprint_demos.cpp - :lines: 468-505 - :language: cpp - :dedent: 4 +.. literalinclude:: t_conduit_docs_blueprint_demos_out.txt + :start-after: BEGIN_EXAMPLE("blueprint_demo_basic_hexs") + :end-before: END_EXAMPLE("blueprint_demo_basic_hexs") + :language: yaml * **Visual** @@ -829,16 +837,17 @@ Polyhedra * **Usage Example** .. literalinclude:: ../../tests/docs/t_conduit_docs_blueprint_demos.cpp - :lines: 567-572 + :start-after: BEGIN_EXAMPLE("blueprint_demo_basic_polyhedra") + :end-before: END_EXAMPLE("blueprint_demo_basic_polyhedra") :language: cpp :dedent: 4 * **Result** -.. literalinclude:: ../../tests/docs/t_conduit_docs_blueprint_demos.cpp - :lines: 575-612 - :language: cpp - :dedent: 4 +.. literalinclude:: t_conduit_docs_blueprint_demos_out.txt + :start-after: BEGIN_EXAMPLE("blueprint_demo_basic_polyhedra") + :end-before: END_EXAMPLE("blueprint_demo_basic_polyhedra") + :language: yaml * **Visual** @@ -1062,7 +1071,8 @@ This snippet provides a complete C++ example that demonstrates: * Saving the result to a JSON file that VisIt can open .. literalinclude:: ../../tests/docs/t_conduit_docs_blueprint_demos.cpp - :lines: 621-680 + :start-after: BEGIN_EXAMPLE("blueprint_demo_basic_uniform_detailed") + :end-before: END_EXAMPLE("blueprint_demo_basic_uniform_detailed") :language: cpp :dedent: 4 diff --git a/src/docs/sphinx/relay_io.rst b/src/docs/sphinx/relay_io.rst index f57f3056a..0cb5aaa2c 100644 --- a/src/docs/sphinx/relay_io.rst +++ b/src/docs/sphinx/relay_io.rst @@ -91,14 +91,16 @@ Save and Load * **C++ Example:** .. literalinclude:: ../../tests/docs/t_conduit_docs_relay_io_generic_examples.cpp - :lines: 93-107 + :start-after: BEGIN_EXAMPLE("relay_io_example_1_json") + :end-before: END_EXAMPLE("relay_io_example_1_json") :language: cpp :dedent: 4 * **Output:** .. literalinclude:: t_conduit_docs_relay_io_generic_examples_out.txt - :lines: 41-65 + :start-after: BEGIN_EXAMPLE("relay_io_example_1_json") + :end-before: END_EXAMPLE("relay_io_example_1_json") Save Merged @@ -107,14 +109,16 @@ Save Merged * **C++ Example:** .. literalinclude:: ../../tests/docs/t_conduit_docs_relay_io_generic_examples.cpp - :lines: 119-140 + :start-after: BEGIN_EXAMPLE("relay_io_example_2_hdf5") + :end-before: END_EXAMPLE("relay_io_example_2_hdf5") :language: cpp :dedent: 4 * **Output:** .. literalinclude:: t_conduit_docs_relay_io_generic_examples_out.txt - :lines: 73-110 + :start-after: BEGIN_EXAMPLE("relay_io_example_2_hdf5") + :end-before: END_EXAMPLE("relay_io_example_2_hdf5") Load Merged @@ -123,14 +127,16 @@ Load Merged * **C++ Example:** .. literalinclude:: ../../tests/docs/t_conduit_docs_relay_io_generic_examples.cpp - :lines: 150-168 + :start-after: BEGIN_EXAMPLE("relay_io_example_3_hdf5") + :end-before: END_EXAMPLE("relay_io_example_3_hdf5") :language: cpp :dedent: 4 * **Output:** .. literalinclude:: t_conduit_docs_relay_io_generic_examples_out.txt - :lines: 118-155 + :start-after: BEGIN_EXAMPLE("relay_io_example_3_hdf5") + :end-before: END_EXAMPLE("relay_io_example_3_hdf5") Load from Subpath +++++++++++++++++++ @@ -138,14 +144,16 @@ Load from Subpath * **C++ Example:** .. literalinclude:: ../../tests/docs/t_conduit_docs_relay_io_generic_examples.cpp - :lines: 180-193 + :start-after: BEGIN_EXAMPLE("relay_io_example_4_hdf5") + :end-before: END_EXAMPLE("relay_io_example_4_hdf5") :language: cpp :dedent: 4 * **Output:** .. literalinclude:: t_conduit_docs_relay_io_generic_examples_out.txt - :lines: 163-179 + :start-after: BEGIN_EXAMPLE("relay_io_example_4_hdf5") + :end-before: END_EXAMPLE("relay_io_example_4_hdf5") Save to Subpath @@ -154,14 +162,16 @@ Save to Subpath * **C++ Example:** .. literalinclude:: ../../tests/docs/t_conduit_docs_relay_io_generic_examples.cpp - :lines: 204-217 + :start-after: BEGIN_EXAMPLE("relay_io_example_5_hdf5") + :end-before: END_EXAMPLE("relay_io_example_5_hdf5") :language: cpp :dedent: 4 * **Output:** .. literalinclude:: t_conduit_docs_relay_io_generic_examples_out.txt - :lines: 187-203 + :start-after: BEGIN_EXAMPLE("relay_io_example_5_hdf5") + :end-before: END_EXAMPLE("relay_io_example_5_hdf5") .. things not yet covered: options @@ -228,28 +238,33 @@ Relay I/O Handle Examples * **C++ Example:** .. literalinclude:: ../../tests/docs/t_conduit_docs_relay_io_handle_examples.cpp - :lines: 64-132 + :start-after: BEGIN_EXAMPLE("relay_io_handle_example_1") + :end-before: END_EXAMPLE("relay_io_handle_example_1") :language: cpp :dedent: 4 * **Output:** .. literalinclude:: t_conduit_docs_relay_io_handle_examples_out.txt - :lines: 9-45 + :start-after: BEGIN_EXAMPLE("relay_io_handle_example_1") + :end-before: END_EXAMPLE("relay_io_handle_example_1") * **Python Example:** .. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_python_relay_io_handle_examples.py - :lines: 65-111 + :start-after: BEGIN_EXAMPLE("py_relay_io_handle") + :end-before: END_EXAMPLE("py_relay_io_handle") :language: python :dedent: 8 * **Output:** -.. literalinclude:: t_conduit_docs_python_tutorial_relay_io_handle_examples_out.txt - :lines: 28-64 +.. literalinclude:: t_conduit_docs_tutorial_python_relay_io_handle_examples_out.txt + :start-after: BEGIN_EXAMPLE("py_relay_io_handle") + :end-before: END_EXAMPLE("py_relay_io_handle") :dedent: 4 + Relay I/O HDF5 Interface --------------------------- @@ -270,14 +285,18 @@ HDF5 I/O Interface Basics * **C++ Example:** .. literalinclude:: ../../tests/docs/t_conduit_docs_relay_io_hdf5_examples.cpp - :lines: 66-115 + :start-after: BEGIN_EXAMPLE("relay_io_example_hdf5_interface_1") + :end-before: END_EXAMPLE("relay_io_example_hdf5_interface_1") :language: cpp :dedent: 4 + + * **Output:** .. literalinclude:: t_conduit_docs_relay_io_hdf5_examples_out.txt - :lines: 9-41 + :start-after: BEGIN_EXAMPLE("relay_io_example_hdf5_interface_1") + :end-before: END_EXAMPLE("relay_io_example_hdf5_interface_1") @@ -287,14 +306,16 @@ HDF5 I/O Options * **C++ Example:** .. literalinclude:: ../../tests/docs/t_conduit_docs_relay_io_hdf5_examples.cpp - :lines: 128-160 + :start-after: BEGIN_EXAMPLE("relay_io_example_hdf5_interface_opts") + :end-before: END_EXAMPLE("relay_io_example_hdf5_interface_opts") :language: cpp :dedent: 4 * **Output:** .. literalinclude:: t_conduit_docs_relay_io_hdf5_examples_out.txt - :lines: 49-107 + :start-after: BEGIN_EXAMPLE("relay_io_example_hdf5_interface_opts") + :end-before: END_EXAMPLE("relay_io_example_hdf5_interface_opts") You can verify using ``h5stat`` that the data set was written to the hdf5 file using chunking and compression. diff --git a/src/docs/sphinx/relay_mpi.rst b/src/docs/sphinx/relay_mpi.rst index 7903067ee..16cf8a35a 100644 --- a/src/docs/sphinx/relay_mpi.rst +++ b/src/docs/sphinx/relay_mpi.rst @@ -154,7 +154,7 @@ Send and Receive Using Schema * **Output:** -.. literalinclude:: t_conduit_docs_python_tutorial_relay_mpi_examples_out.txt +.. literalinclude:: t_conduit_docs_tutorial_python_relay_mpi_examples_out.txt :start-after: BEGIN_EXAMPLE("py_mpi_send_and_recv_using_schema") :end-before: END_EXAMPLE("py_mpi_send_and_recv_using_schema") :dedent: 4 @@ -166,15 +166,15 @@ Send and Receive * **Python Source:** .. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_python_relay_mpi_examples.py - :start-after: BEGIN_EXAMPLE("py_mpi_send_and_recv")) + :start-after: BEGIN_EXAMPLE("py_mpi_send_and_recv") :end-before: END_EXAMPLE("py_mpi_send_and_recv") :language: python :dedent: 8 * **Output:** -.. literalinclude:: t_conduit_docs_python_tutorial_relay_mpi_examples_out.txt - :start-after: BEGIN_EXAMPLE("py_mpi_send_and_recv")) +.. literalinclude:: t_conduit_docs_tutorial_python_relay_mpi_examples_out.txt + :start-after: BEGIN_EXAMPLE("py_mpi_send_and_recv") :end-before: END_EXAMPLE("py_mpi_send_and_recv") :dedent: 4 @@ -191,7 +191,7 @@ Send and Receive * **Output:** -.. literalinclude:: t_conduit_docs_python_tutorial_relay_mpi_examples_out.txt +.. literalinclude:: t_conduit_docs_tutorial_python_relay_mpi_examples_out.txt :start-after: BEGIN_EXAMPLE("py_mpi_send_and_recv") :end-before: END_EXAMPLE("py_mpi_send_and_recv") :dedent: 4 @@ -209,7 +209,7 @@ Sum All Reduce * **Output:** -.. literalinclude:: t_conduit_docs_python_tutorial_relay_mpi_examples_out.txt +.. literalinclude:: t_conduit_docs_tutorial_python_relay_mpi_examples_out.txt :start-after: BEGIN_EXAMPLE("py_mpi_sum_all_reduce") :end-before: END_EXAMPLE("py_mpi_sum_all_reduce") :dedent: 4 @@ -227,7 +227,7 @@ Broadcast Using Schema * **Output:** -.. literalinclude:: t_conduit_docs_python_tutorial_relay_mpi_examples_out.txt +.. literalinclude:: t_conduit_docs_tutorial_python_relay_mpi_examples_out.txt :start-after: BEGIN_EXAMPLE("py_mpi_bcast_using_schema") :end-before: END_EXAMPLE("py_mpi_bcast_using_schema") :dedent: 4 @@ -239,15 +239,15 @@ Broadcast * **Python Source:** .. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_python_relay_mpi_examples.py - :start-after: BEGIN_EXAMPLE("py_mpi_bcast")) + :start-after: BEGIN_EXAMPLE("py_mpi_bcast") :end-before: END_EXAMPLE("py_mpi_bcast") :language: python :dedent: 8 * **Output:** -.. literalinclude:: t_conduit_docs_python_tutorial_relay_mpi_examples_out.txt - :start-after: BEGIN_EXAMPLE("py_mpi_bcast")) +.. literalinclude:: t_conduit_docs_tutorial_python_relay_mpi_examples_out.txt + :start-after: BEGIN_EXAMPLE("py_mpi_bcast") :end-before: END_EXAMPLE("py_mpi_bcast") :dedent: 4 @@ -265,7 +265,7 @@ All Gather Using Schema * **Output:** -.. literalinclude:: t_conduit_docs_python_tutorial_relay_mpi_examples_out.txt +.. literalinclude:: t_conduit_docs_tutorial_python_relay_mpi_examples_out.txt :start-after: BEGIN_EXAMPLE("py_mpi_all_gather_using_schema") :end-before: END_EXAMPLE("py_mpi_all_gather_using_schema") :dedent: 4 diff --git a/src/docs/sphinx/t_conduit_docs_blueprint_demos_out.txt b/src/docs/sphinx/t_conduit_docs_blueprint_demos_out.txt new file mode 100644 index 000000000..19a65aea8 --- /dev/null +++ b/src/docs/sphinx/t_conduit_docs_blueprint_demos_out.txt @@ -0,0 +1,287 @@ +Running main() from /Users/harrison37/Work/github/llnl/conduit/src/blt/thirdparty_builtin/googletest-master-2020-01-07/googletest/src/gtest_main.cc +[==========] Running 10 tests from 1 test suite. +[----------] Global test environment set-up. +[----------] 10 tests from conduit_docs +[ RUN ] conduit_docs.blueprint_demo_basic_uniform +BEGIN_EXAMPLE("blueprint_demo_basic_uniform") + +coordsets: + coords: + type: "uniform" + dims: + i: 3 + j: 3 + origin: + x: -10.0 + y: -10.0 + spacing: + dx: 10.0 + dy: 10.0 +topologies: + mesh: + type: "uniform" + coordset: "coords" +fields: + field: + association: "element" + topology: "mesh" + volume_dependent: "false" + values: [0.0, 1.0, 2.0, 3.0] + +END_EXAMPLE("blueprint_demo_basic_uniform") +[/Users/harrison37/Work/github/llnl/conduit/src/tests/docs/t_conduit_docs_blueprint_demos.cpp : 107] + Testing Basic Example 'uniform' +[ OK ] conduit_docs.blueprint_demo_basic_uniform (8 ms) +[ RUN ] conduit_docs.blueprint_demo_basic_rectilinear +BEGIN_EXAMPLE("blueprint_demo_basic_rectilinear") + +coordsets: + coords: + type: "rectilinear" + values: + x: [-10.0, 0.0, 10.0] + y: [-10.0, 0.0, 10.0] +topologies: + mesh: + type: "rectilinear" + coordset: "coords" +fields: + field: + association: "element" + topology: "mesh" + volume_dependent: "false" + values: [0.0, 1.0, 2.0, 3.0] + +END_EXAMPLE("blueprint_demo_basic_rectilinear") +[/Users/harrison37/Work/github/llnl/conduit/src/tests/docs/t_conduit_docs_blueprint_demos.cpp : 107] + Testing Basic Example 'rectilinear' +[ OK ] conduit_docs.blueprint_demo_basic_rectilinear (5 ms) +[ RUN ] conduit_docs.blueprint_demo_basic_structured +BEGIN_EXAMPLE("blueprint_demo_basic_structured") + +coordsets: + coords: + type: "explicit" + values: + x: [-10.0, 0.0, 10.0, -10.0, 0.0, 10.0, -10.0, 0.0, 10.0] + y: [-10.0, -10.0, -10.0, 0.0, 0.0, 0.0, 10.0, 10.0, 10.0] +topologies: + mesh: + type: "structured" + coordset: "coords" + elements: + dims: + i: 2 + j: 2 +fields: + field: + association: "element" + topology: "mesh" + volume_dependent: "false" + values: [0.0, 1.0, 2.0, 3.0] + +END_EXAMPLE("blueprint_demo_basic_structured") +[/Users/harrison37/Work/github/llnl/conduit/src/tests/docs/t_conduit_docs_blueprint_demos.cpp : 107] + Testing Basic Example 'structured' +[ OK ] conduit_docs.blueprint_demo_basic_structured (8 ms) +[ RUN ] conduit_docs.blueprint_demo_basic_tris +BEGIN_EXAMPLE("blueprint_demo_basic_tris") + +coordsets: + coords: + type: "explicit" + values: + x: [-10.0, 0.0, 10.0, -10.0, 0.0, 10.0, -10.0, 0.0, 10.0] + y: [-10.0, -10.0, -10.0, 0.0, 0.0, 0.0, 10.0, 10.0, 10.0] +topologies: + mesh: + type: "unstructured" + coordset: "coords" + elements: + shape: "tri" + connectivity: [0, 3, 4, 0, 1, 4, 1, 4, 5, 1, 2, 5, 3, 6, 7, 3, 4, 7, 4, 7, 8, 4, 5, 8] +fields: + field: + association: "element" + topology: "mesh" + volume_dependent: "false" + values: [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0] + +END_EXAMPLE("blueprint_demo_basic_tris") +[/Users/harrison37/Work/github/llnl/conduit/src/tests/docs/t_conduit_docs_blueprint_demos.cpp : 107] + Testing Basic Example 'tris' +[ OK ] conduit_docs.blueprint_demo_basic_tris (6 ms) +[ RUN ] conduit_docs.blueprint_demo_basic_quads +BEGIN_EXAMPLE("blueprint_demo_basic_quads") + +coordsets: + coords: + type: "explicit" + values: + x: [-10.0, 0.0, 10.0, -10.0, 0.0, 10.0, -10.0, 0.0, 10.0] + y: [-10.0, -10.0, -10.0, 0.0, 0.0, 0.0, 10.0, 10.0, 10.0] +topologies: + mesh: + type: "unstructured" + coordset: "coords" + elements: + shape: "quad" + connectivity: [0, 3, 4, 1, 1, 4, 5, 2, 3, 6, 7, 4, 4, 7, 8, 5] +fields: + field: + association: "element" + topology: "mesh" + volume_dependent: "false" + values: [0.0, 1.0, 2.0, 3.0] + +END_EXAMPLE("blueprint_demo_basic_quads") +[/Users/harrison37/Work/github/llnl/conduit/src/tests/docs/t_conduit_docs_blueprint_demos.cpp : 107] + Testing Basic Example 'quads' +[ OK ] conduit_docs.blueprint_demo_basic_quads (6 ms) +[ RUN ] conduit_docs.blueprint_demo_basic_tets +BEGIN_EXAMPLE("blueprint_demo_basic_tets") + +coordsets: + coords: + type: "explicit" + values: + x: [-10.0, 0.0, 10.0, -10.0, 0.0, 10.0, -10.0, 0.0, 10.0, -10.0, 0.0, 10.0, -10.0, 0.0, 10.0, -10.0, 0.0, 10.0, -10.0, 0.0, 10.0, -10.0, 0.0, 10.0, -10.0, 0.0, 10.0] + y: [-10.0, -10.0, -10.0, 0.0, 0.0, 0.0, 10.0, 10.0, 10.0, -10.0, -10.0, -10.0, 0.0, 0.0, 0.0, 10.0, 10.0, 10.0, -10.0, -10.0, -10.0, 0.0, 0.0, 0.0, 10.0, 10.0, 10.0] + z: [-10.0, -10.0, -10.0, -10.0, -10.0, -10.0, -10.0, -10.0, -10.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0] +topologies: + mesh: + type: "unstructured" + coordset: "coords" + elements: + shape: "tet" + connectivity: [0, 4, 1, 13, 0, 3, 4, 13, 0, 12, 3, 13, 0, 9, 12, 13, 0, 10, 9, 13, 0, 1, 10, 13, 1, 5, 2, 14, 1, 4, 5, 14, 1, 13, 4, 14, 1, 10, 13, 14, 1, 11, 10, 14, 1, 2, 11, 14, 3, 7, 4, 16, 3, 6, 7, 16, 3, 15, 6, 16, 3, 12, 15, 16, 3, 13, 12, 16, 3, 4, 13, 16, 4, 8, 5, 17, 4, 7, 8, 17, 4, 16, 7, 17, 4, 13, 16, 17, 4, 14, 13, 17, 4, 5, 14, 17, 9, 13, 10, 22, 9, 12, 13, 22, 9, 21, 12, 22, 9, 18, 21, 22, 9, 19, 18, 22, 9, 10, 19, 22, 10, 14, 11, 23, 10, 13, 14, 23, 10, 22, 13, 23, 10, 19, 22, 23, 10, 20, 19, 23, 10, 11, 20, 23, 12, 16, 13, 25, 12, 15, 16, 25, 12, 24, 15, 25, 12, 21, 24, 25, 12, 22, 21, 25, 12, 13, 22, 25, 13, 17, 14, 26, 13, 16, 17, 26, 13, 25, 16, 26, 13, 22, 25, 26, 13, 23, 22, 26, 13, 14, 23, 26] +fields: + field: + association: "element" + topology: "mesh" + volume_dependent: "false" + values: [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, 30.0, 31.0, 32.0, 33.0, 34.0, 35.0, 36.0, 37.0, 38.0, 39.0, 40.0, 41.0, 42.0, 43.0, 44.0, 45.0, 46.0, 47.0] + +END_EXAMPLE("blueprint_demo_basic_tets") +[/Users/harrison37/Work/github/llnl/conduit/src/tests/docs/t_conduit_docs_blueprint_demos.cpp : 107] + Testing Basic Example 'tets' +[ OK ] conduit_docs.blueprint_demo_basic_tets (7 ms) +[ RUN ] conduit_docs.blueprint_demo_basic_hexs +BEGIN_EXAMPLE("blueprint_demo_basic_hexs") + +coordsets: + coords: + type: "explicit" + values: + x: [-10.0, 0.0, 10.0, -10.0, 0.0, 10.0, -10.0, 0.0, 10.0, -10.0, 0.0, 10.0, -10.0, 0.0, 10.0, -10.0, 0.0, 10.0, -10.0, 0.0, 10.0, -10.0, 0.0, 10.0, -10.0, 0.0, 10.0] + y: [-10.0, -10.0, -10.0, 0.0, 0.0, 0.0, 10.0, 10.0, 10.0, -10.0, -10.0, -10.0, 0.0, 0.0, 0.0, 10.0, 10.0, 10.0, -10.0, -10.0, -10.0, 0.0, 0.0, 0.0, 10.0, 10.0, 10.0] + z: [-10.0, -10.0, -10.0, -10.0, -10.0, -10.0, -10.0, -10.0, -10.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0] +topologies: + mesh: + type: "unstructured" + coordset: "coords" + elements: + shape: "hex" + connectivity: [0, 1, 4, 3, 9, 10, 13, 12, 1, 2, 5, 4, 10, 11, 14, 13, 3, 4, 7, 6, 12, 13, 16, 15, 4, 5, 8, 7, 13, 14, 17, 16, 9, 10, 13, 12, 18, 19, 22, 21, 10, 11, 14, 13, 19, 20, 23, 22, 12, 13, 16, 15, 21, 22, 25, 24, 13, 14, 17, 16, 22, 23, 26, 25] +fields: + field: + association: "element" + topology: "mesh" + volume_dependent: "false" + values: [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0] + +END_EXAMPLE("blueprint_demo_basic_hexs") +[/Users/harrison37/Work/github/llnl/conduit/src/tests/docs/t_conduit_docs_blueprint_demos.cpp : 107] + Testing Basic Example 'hexs' +[ OK ] conduit_docs.blueprint_demo_basic_hexs (6 ms) +[ RUN ] conduit_docs.blueprint_demo_basic_polygons +BEGIN_EXAMPLE("blueprint_demo_basic_polygons") + +coordsets: + coords: + type: "explicit" + values: + x: [-10.0, 0.0, 10.0, -10.0, 0.0, 10.0, -10.0, 0.0, 10.0] + y: [-10.0, -10.0, -10.0, 0.0, 0.0, 0.0, 10.0, 10.0, 10.0] +topologies: + mesh: + type: "unstructured" + coordset: "coords" + elements: + shape: "polygonal" + connectivity: [4, 0, 3, 4, 1, 4, 1, 4, 5, 2, 4, 3, 6, 7, 4, 4, 4, 7, 8, 5] +fields: + field: + association: "element" + topology: "mesh" + volume_dependent: "false" + values: [0.0, 1.0, 2.0, 3.0] + +END_EXAMPLE("blueprint_demo_basic_polygons") +[/Users/harrison37/Work/github/llnl/conduit/src/tests/docs/t_conduit_docs_blueprint_demos.cpp : 107] + Testing Basic Example 'polygons' +[ OK ] conduit_docs.blueprint_demo_basic_polygons (6 ms) +[ RUN ] conduit_docs.blueprint_demo_basic_polyhedra +BEGIN_EXAMPLE("blueprint_demo_basic_polyhedra") + +coordsets: + coords: + type: "explicit" + values: + x: [-10.0, 0.0, 10.0, -10.0, 0.0, 10.0, -10.0, 0.0, 10.0, -10.0, 0.0, 10.0, -10.0, 0.0, 10.0, -10.0, 0.0, 10.0, -10.0, 0.0, 10.0, -10.0, 0.0, 10.0, -10.0, 0.0, 10.0] + y: [-10.0, -10.0, -10.0, 0.0, 0.0, 0.0, 10.0, 10.0, 10.0, -10.0, -10.0, -10.0, 0.0, 0.0, 0.0, 10.0, 10.0, 10.0, -10.0, -10.0, -10.0, 0.0, 0.0, 0.0, 10.0, 10.0, 10.0] + z: [-10.0, -10.0, -10.0, -10.0, -10.0, -10.0, -10.0, -10.0, -10.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0] +topologies: + mesh: + type: "unstructured" + coordset: "coords" + elements: + shape: "polyhedral" + connectivity: [6, 4, 0, 1, 4, 3, 4, 0, 1, 10, 9, 4, 1, 4, 13, 10, 4, 4, 3, 12, 13, 4, 3, 0, 9, 12, 4, 9, 10, 13, 12, 6, 4, 1, 2, 5, 4, 4, 1, 2, 11, 10, 4, 2, 5, 14, 11, 4, 5, 4, 13, 14, 4, 4, 1, 10, 13, 4, 10, 11, 14, 13, 6, 4, 3, 4, 7, 6, 4, 3, 4, 13, 12, 4, 4, 7, 16, 13, 4, 7, 6, 15, 16, 4, 6, 3, 12, 15, 4, 12, 13, 16, 15, 6, 4, 4, 5, 8, 7, 4, 4, 5, 14, 13, 4, 5, 8, 17, 14, 4, 8, 7, 16, 17, 4, 7, 4, 13, 16, 4, 13, 14, 17, 16, 6, 4, 9, 10, 13, 12, 4, 9, 10, 19, 18, 4, 10, 13, 22, 19, 4, 13, 12, 21, 22, 4, 12, 9, 18, 21, 4, 18, 19, 22, 21, 6, 4, 10, 11, 14, 13, 4, 10, 11, 20, 19, 4, 11, 14, 23, 20, 4, 14, 13, 22, 23, 4, 13, 10, 19, 22, 4, 19, 20, 23, 22, 6, 4, 12, 13, 16, 15, 4, 12, 13, 22, 21, 4, 13, 16, 25, 22, 4, 16, 15, 24, 25, 4, 15, 12, 21, 24, 4, 21, 22, 25, 24, 6, 4, 13, 14, 17, 16, 4, 13, 14, 23, 22, 4, 14, 17, 26, 23, 4, 17, 16, 25, 26, 4, 16, 13, 22, 25, 4, 22, 23, 26, 25] +fields: + field: + association: "element" + topology: "mesh" + volume_dependent: "false" + values: [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0] + +END_EXAMPLE("blueprint_demo_basic_polyhedra") +[/Users/harrison37/Work/github/llnl/conduit/src/tests/docs/t_conduit_docs_blueprint_demos.cpp : 107] + Testing Basic Example 'polyhedra' +[ OK ] conduit_docs.blueprint_demo_basic_polyhedra (8 ms) +[ RUN ] conduit_docs.blueprint_demo_basic_uniform_detailed +BEGIN_EXAMPLE("blueprint_demo_basic_uniform_detailed") + +coordsets: + coords: + type: "uniform" + dims: + i: 3 + j: 3 + origin: + x: -10.0 + y: -10.0 + spacing: + dx: 10.0 + dy: 10.0 +topologies: + topo: + type: "uniform" + coordset: "coords" +fields: + ele_example: + association: "element" + topology: "topo" + values: [0.0, 1.0, 2.0, 3.0] + vert_example: + association: "vertex" + topology: "topo" + values: [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0] + +END_EXAMPLE("blueprint_demo_basic_uniform_detailed") +[ OK ] conduit_docs.blueprint_demo_basic_uniform_detailed (2 ms) +[----------] 10 tests from conduit_docs (62 ms total) + +[----------] Global test environment tear-down +[==========] 10 tests from 1 test suite ran. (62 ms total) +[ PASSED ] 10 tests. diff --git a/src/docs/sphinx/t_conduit_docs_blueprint_examples_out.txt b/src/docs/sphinx/t_conduit_docs_blueprint_examples_out.txt index 89a3c0dcb..944789b81 100644 --- a/src/docs/sphinx/t_conduit_docs_blueprint_examples_out.txt +++ b/src/docs/sphinx/t_conduit_docs_blueprint_examples_out.txt @@ -1,63 +1,51 @@ -Running main() from gtest_main.cc -[==========] Running 2 tests from 1 test case. +Running main() from /Users/harrison37/Work/github/llnl/conduit/src/blt/thirdparty_builtin/googletest-master-2020-01-07/googletest/src/gtest_main.cc +[==========] Running 2 tests from 1 test suite. [----------] Global test environment set-up. [----------] 2 tests from conduit_docs [ RUN ] conduit_docs.blueprint_example_1 -[/Users/harrison37/Work/conduit/src/tests/docs/t_conduit_docs_blueprint_examples.cpp : 63] - blueprint_example_1 +BEGIN_EXAMPLE("blueprint_example_1") mesh verify succeeded. -{ - "coords": - { - "values": - { - "valid": "true" - }, - "valid": "true" - } -} -[/Users/harrison37/Work/conduit/src/tests/docs/t_conduit_docs_blueprint_examples.cpp : 81] - blueprint_example_1 +valid: "true" +coords: + type: + valid: "true" + info: + - "mesh::coordset::explicit: 'type' has valid value 'explicit'" + - "mesh::coordset::explicit: 'values' is an mcarray" + values: + valid: "true" + valid: "true" + +END_EXAMPLE("blueprint_example_1") [ OK ] conduit_docs.blueprint_example_1 (1 ms) [ RUN ] conduit_docs.blueprint_example_2 -[/Users/harrison37/Work/conduit/src/tests/docs/t_conduit_docs_blueprint_examples.cpp : 88] - blueprint_example_2 +BEGIN_EXAMPLE("blueprint_example_2") example 'separate' mcarray -{ - "x": [1.0, 1.0, 1.0, 1.0, 1.0], - "y": [2.0, 2.0, 2.0, 2.0, 2.0], - "z": [3.0, 3.0, 3.0, 3.0, 3.0] -} +x: [1.0, 1.0, 1.0, 1.0, 1.0] +y: [2.0, 2.0, 2.0, 2.0, 2.0] +z: [3.0, 3.0, 3.0, 3.0, 3.0] + + +mem_spaces: + 0x7fc12cc04e20: + path: "x" + type: "allocated" + bytes: 40 + 0x7fc12cc04f20: + path: "y" + type: "allocated" + bytes: 40 + 0x7fc12cc05080: + path: "z" + type: "allocated" + bytes: 40 +total_bytes_allocated: 120 +total_bytes_mmaped: 0 +total_bytes_compact: 120 +total_strided_bytes: 120 -{ - "mem_spaces": - { - "0x7fd6c0600100": - { - "path": "x", - "type": "allocated", - "bytes": 40 - }, - "0x7fd6c0600460": - { - "path": "y", - "type": "allocated", - "bytes": 40 - }, - "0x7fd6c0600130": - { - "path": "z", - "type": "allocated", - "bytes": 40 - } - }, - "total_bytes_allocated": 120, - "total_bytes_mmaped": 0, - "total_bytes_compact": 120, - "total_strided_bytes": 120 -} transformed to 'interleaved' mcarray { @@ -66,26 +54,20 @@ transformed to 'interleaved' mcarray "z": {"dtype":"float64", "number_of_elements": 5, "offset": 16, "stride": 24, "element_bytes": 8, "endianness": "little", "value": [3.0, 3.0, 3.0, 3.0, 3.0]} } -{ - "mem_spaces": - { - "0x7fd6c0602090": - { - "path": "", - "type": "allocated", - "bytes": 120 - } - }, - "total_bytes_allocated": 120, - "total_bytes_mmaped": 0, - "total_bytes_compact": 120, - "total_strided_bytes": 312 -} -[/Users/harrison37/Work/conduit/src/tests/docs/t_conduit_docs_blueprint_examples.cpp : 117] - blueprint_example_2 -[ OK ] conduit_docs.blueprint_example_2 (0 ms) -[----------] 2 tests from conduit_docs (1 ms total) +mem_spaces: + 0x7fc12cc06b70: + path: "" + type: "allocated" + bytes: 120 +total_bytes_allocated: 120 +total_bytes_mmaped: 0 +total_bytes_compact: 120 +total_strided_bytes: 312 + +END_EXAMPLE("blueprint_example_2") +[ OK ] conduit_docs.blueprint_example_2 (1 ms) +[----------] 2 tests from conduit_docs (2 ms total) [----------] Global test environment tear-down -[==========] 2 tests from 1 test case ran. (1 ms total) +[==========] 2 tests from 1 test suite ran. (2 ms total) [ PASSED ] 2 tests. diff --git a/src/docs/sphinx/t_conduit_docs_python_tutorial_basics_out.txt b/src/docs/sphinx/t_conduit_docs_python_tutorial_basics_out.txt deleted file mode 100644 index 2f9e14f3f..000000000 --- a/src/docs/sphinx/t_conduit_docs_python_tutorial_basics_out.txt +++ /dev/null @@ -1,162 +0,0 @@ -UpdateCTestConfiguration from :/Users/harrison37/Work/conduit/build-debug/DartConfiguration.tcl -Parse Config file:/Users/harrison37/Work/conduit/build-debug/DartConfiguration.tcl -UpdateCTestConfiguration from :/Users/harrison37/Work/conduit/build-debug/DartConfiguration.tcl -Parse Config file:/Users/harrison37/Work/conduit/build-debug/DartConfiguration.tcl -Test project /Users/harrison37/Work/conduit/build-debug -Constructing a list of tests -Done constructing a list of tests -Updating test list for fixtures -Added 0 tests to meet fixture requirements -Checking test dependency graph... -Checking test dependency graph end -test 87 - Start 87: t_conduit_docs_tutorial_python_basics - -87: Test command: /Users/harrison37/Work/conduit/uberenv_libs/spack/opt/spack/darwin-elcapitan-x86_64/clang-7.3.0-apple/python-2.7.14-hnuoilo6bgembdfjyozsi5sztwgnnwac/bin/python2.7 "-B" "-m" "unittest" "-v" "t_conduit_docs_tutorial_python_basics" -87: Environment variables: -87: PYTHONPATH=/Users/harrison37/Work/conduit/build-debug/python-modules/:/Users/harrison37/Work/conduit/src/tests/docs -87: Test timeout computed to be: 1500 -87: test_001_basics_very_basic (t_conduit_docs_tutorial_python_basics.Conduit_Tutorial_Python_Basics) ... ok -87: test_002_basics_hierarchial (t_conduit_docs_tutorial_python_basics.Conduit_Tutorial_Python_Basics) ... ok -87: test_003_basics_object_and_list (t_conduit_docs_tutorial_python_basics.Conduit_Tutorial_Python_Basics) ... ok -87: test_004_basics_object_and_list_itr (t_conduit_docs_tutorial_python_basics.Conduit_Tutorial_Python_Basics) ... ok -87: test_005_basics_mem_spaces (t_conduit_docs_tutorial_python_basics.Conduit_Tutorial_Python_Basics) ... ok -87: test_006_basics_bw_style (t_conduit_docs_tutorial_python_basics.Conduit_Tutorial_Python_Basics) ... ok -87: test_007_basics_bw_style_from_native (t_conduit_docs_tutorial_python_basics.Conduit_Tutorial_Python_Basics) ... -87: begin: test_001_basics_very_basic,60 -87: -87: { -87: "my": "data" -87: } -87: -87: end: test_001_basics_very_basic,66 -87: -87: begin: test_002_basics_hierarchial,70 -87: -87: { -87: "my": "data", -87: "a": -87: { -87: "b": -87: { -87: "c": "d", -87: "e": 64.0 -87: } -87: } -87: } -87: total bytes: 15 -87: -87: -87: end: test_002_basics_hierarchial,79 -87: -87: begin: test_003_basics_object_and_list,82 -87: -87: { -87: "object_example": -87: { -87: "val1": "data", -87: "val2": 10, -87: "val3": 3.1415 -87: }, -87: "list_example": -87: [ -87: 0, -87: 1, -87: 2, -87: 3, -87: 4 -87: ] -87: } -87: -87: end: test_003_basics_object_and_list,94 -87: -87: begin: test_004_basics_object_and_list_itr,97 -87: -87: { -87: "object_example": -87: { -87: "val1": "data", -87: "val2": 10, -87: "val3": 3.1415 -87: }, -87: "list_example": -87: [ -87: 0, -87: 1, -87: 2, -87: 3, -87: 4 -87: ] -87: } -87: val1: "data" -87: val2: 10 -87: val3: 3.1415 -87: 0 -87: 1 -87: 2 -87: 3 -87: 4 -87: -87: end: test_004_basics_object_and_list_itr,115 -87: -87: begin: test_005_basics_mem_spaces,118 -87: -87: { -87: "mem_spaces": -87: { -87: "0x7fb538d51320": -87: { -87: "path": "my", -87: "type": "allocated", -87: "bytes": 5 -87: }, -87: "0x7fb538d31db0": -87: { -87: "path": "a/b/c", -87: "type": "allocated", -87: "bytes": 2 -87: }, -87: "0x7fb538daf890": -87: { -87: "path": "a/b/e", -87: "type": "allocated", -87: "bytes": 8 -87: } -87: }, -87: "total_bytes_allocated": 15, -87: "total_bytes_mmaped": 0, -87: "total_bytes_compact": 15, -87: "total_strided_bytes": 15 -87: } -87: -87: end: test_005_basics_mem_spaces,126 -87: -87: begin: test_006_basics_bw_style,129 -87: -87: { -87: "test": 100 -87: } -87: -87: end: test_006_basics_bw_style,135 -87: -87: begin: test_007_basics_bw_style_from_native,139 -87: -87: { -87: "test": {"dtype":"int64", "number_of_elements": 1, "offset": 0, "stride": 8, "element_bytes": 8, "endianness": "little", "value": 10} -87: } -87: ok -87: -87: ---------------------------------------------------------------------- -87: Ran 7 tests in 0.030s -87: -87: OK -87: -87: end: test_007_basics_bw_style_from_native,145 -1/1 Test #87: t_conduit_docs_tutorial_python_basics ... Passed 0.14 sec - -The following tests passed: - t_conduit_docs_tutorial_python_basics - -100% tests passed, 0 tests failed out of 1 - -Total Test time (real) = 0.15 sec diff --git a/src/docs/sphinx/t_conduit_docs_python_tutorial_json_out.txt b/src/docs/sphinx/t_conduit_docs_python_tutorial_json_out.txt deleted file mode 100644 index c9179a4de..000000000 --- a/src/docs/sphinx/t_conduit_docs_python_tutorial_json_out.txt +++ /dev/null @@ -1,51 +0,0 @@ -UpdateCTestConfiguration from :/Users/harrison37/Work/conduit/build-debug/DartConfiguration.tcl -Parse Config file:/Users/harrison37/Work/conduit/build-debug/DartConfiguration.tcl -UpdateCTestConfiguration from :/Users/harrison37/Work/conduit/build-debug/DartConfiguration.tcl -Parse Config file:/Users/harrison37/Work/conduit/build-debug/DartConfiguration.tcl -Test project /Users/harrison37/Work/conduit/build-debug -Constructing a list of tests -Done constructing a list of tests -Updating test list for fixtures -Added 0 tests to meet fixture requirements -Checking test dependency graph... -Checking test dependency graph end -test 89 - Start 89: t_conduit_docs_tutorial_python_json - -89: Test command: /Users/harrison37/Work/conduit/uberenv_libs/spack/opt/spack/darwin-elcapitan-x86_64/clang-7.3.0-apple/python-2.7.14-hnuoilo6bgembdfjyozsi5sztwgnnwac/bin/python2.7 "-B" "-m" "unittest" "-v" "t_conduit_docs_tutorial_python_json" -89: Environment variables: -89: PYTHONPATH=/Users/harrison37/Work/conduit/build-debug/python-modules/:/Users/harrison37/Work/conduit/src/tests/docs -89: Test timeout computed to be: 1500 -89: test_001_json_generator_std (t_conduit_docs_tutorial_python_json.Conduit_Tutorial_Python_JSON) ... ok -89: test_002_json_generator_pure_json (t_conduit_docs_tutorial_python_json.Conduit_Tutorial_Python_JSON) ... ok -89: -89: ---------------------------------------------------------------------- -89: Ran 2 tests in 0.024s -89: -89: OK -89: -89: begin: test_001_json_generator_std,60 -89: 100.0 -89: -89: { -89: "test": 100.0 -89: } -89: -89: end: test_001_json_generator_std,69 -89: -89: begin: test_002_json_generator_pure_json,72 -89: 100.0 -89: -89: { -89: "test": 100.0 -89: } -89: -89: end: test_002_json_generator_pure_json,81 -1/1 Test #89: t_conduit_docs_tutorial_python_json ... Passed 0.13 sec - -The following tests passed: - t_conduit_docs_tutorial_python_json - -100% tests passed, 0 tests failed out of 1 - -Total Test time (real) = 0.14 sec diff --git a/src/docs/sphinx/t_conduit_docs_python_tutorial_parse_out.txt b/src/docs/sphinx/t_conduit_docs_python_tutorial_parse_out.txt deleted file mode 100644 index 83a42773a..000000000 --- a/src/docs/sphinx/t_conduit_docs_python_tutorial_parse_out.txt +++ /dev/null @@ -1,76 +0,0 @@ -UpdateCTestConfiguration from :/Users/harrison37/Work/github/llnl/conduit/build-debug/DartConfiguration.tcl -Parse Config file:/Users/harrison37/Work/github/llnl/conduit/build-debug/DartConfiguration.tcl -UpdateCTestConfiguration from :/Users/harrison37/Work/github/llnl/conduit/build-debug/DartConfiguration.tcl -Parse Config file:/Users/harrison37/Work/github/llnl/conduit/build-debug/DartConfiguration.tcl -Test project /Users/harrison37/Work/github/llnl/conduit/build-debug -Constructing a list of tests -Done constructing a list of tests -Updating test list for fixtures -Added 0 tests to meet fixture requirements -Checking test dependency graph... -Checking test dependency graph end -test 104 - Start 104: t_conduit_docs_tutorial_python_parse - -104: Test command: /Users/harrison37/Work/github/llnl/conduit/uberenv_libs/spack/opt/spack/darwin-highsierra-x86_64/clang-9.0.0-apple/python-3.7.4-ss4j2sdo26gewstsqqmjxxzri7ozxevh/bin/python3.7 "-B" "-m" "unittest" "-v" "t_conduit_docs_tutorial_python_parse" -104: Environment variables: -104: PYTHONPATH=/Users/harrison37/Work/github/llnl/conduit/build-debug/python-modules/:/Users/harrison37/Work/github/llnl/conduit/src/tests/docs -104: Test timeout computed to be: 1500 -104: test_docs_tutorial_json (t_conduit_docs_tutorial_python_parse.Conduit_Tutorial_Python_Parse) ... ok -104: test_docs_tutorial_yaml (t_conduit_docs_tutorial_python_parse.Conduit_Tutorial_Python_Parse) ... ok -104: test_docs_tutorial_yaml_inline_array (t_conduit_docs_tutorial_python_parse.Conduit_Tutorial_Python_Parse) ... ok -104: test_json_generator_pure_json (t_conduit_docs_tutorial_python_parse.Conduit_Tutorial_Python_Parse) ... ok -104: test_json_generator_pure_yaml (t_conduit_docs_tutorial_python_parse.Conduit_Tutorial_Python_Parse) ... ok -104: test_json_generator_std (t_conduit_docs_tutorial_python_parse.Conduit_Tutorial_Python_Parse) ... ok -104: -104: ---------------------------------------------------------------------- -104: Ran 6 tests in 0.001s -104: -104: OK -104: BEGIN_EXAMPLE("t_py_conduit_docs_tutorial_json") -104: 42.0 -104: -104: { -104: "mykey": {"dtype":"float64", "number_of_elements": 1, "offset": 0, "stride": 8, "element_bytes": 8, "endianness": "little"} -104: } -104: END_EXAMPLE("t_py_conduit_docs_tutorial_json") -104: BEGIN_EXAMPLE("t_py_conduit_docs_tutorial_yaml") -104: 42.0 -104: -104: { -104: "mykey": {"dtype":"float64", "number_of_elements": 1, "offset": 0, "stride": 8, "element_bytes": 8, "endianness": "little"} -104: } -104: END_EXAMPLE("t_py_conduit_docs_tutorial_yaml") -104: BEGIN_EXAMPLE("t_py_conduit_docs_tutorial_yaml_inline_array") -104: [ 0. 10. 20. 30.] -104: {"dtype":"float64", "number_of_elements": 4, "offset": 0, "stride": 8, "element_bytes": 8, "endianness": "little"} -104: END_EXAMPLE("t_py_conduit_docs_tutorial_yaml_inline_array") -104: BEGIN_EXAMPLE("t_py_json_generator_pure_json") -104: 100.0 -104: -104: { -104: "test": 100.0 -104: } -104: END_EXAMPLE("t_py_json_generator_pure_json") -104: BEGIN_EXAMPLE("t_py_json_generator_pure_yaml") -104: 100.0 -104: -104: { -104: "test": 100.0 -104: } -104: END_EXAMPLE("t_py_json_generator_pure_yaml") -104: BEGIN_EXAMPLE("t_py_json_generator_std") -104: 100.0 -104: -104: { -104: "test": 100.0 -104: } -104: END_EXAMPLE("t_py_json_generator_std") -1/1 Test #104: t_conduit_docs_tutorial_python_parse ... Passed 0.17 sec - -The following tests passed: - t_conduit_docs_tutorial_python_parse - -100% tests passed, 0 tests failed out of 1 - -Total Test time (real) = 0.20 sec diff --git a/src/docs/sphinx/t_conduit_docs_python_tutorial_relay_io_handle_examples_out.txt b/src/docs/sphinx/t_conduit_docs_python_tutorial_relay_io_handle_examples_out.txt deleted file mode 100644 index 30781d707..000000000 --- a/src/docs/sphinx/t_conduit_docs_python_tutorial_relay_io_handle_examples_out.txt +++ /dev/null @@ -1,74 +0,0 @@ -UpdateCTestConfiguration from :/Users/harrison37/Work/github/llnl/conduit/build-debug/DartConfiguration.tcl -Parse Config file:/Users/harrison37/Work/github/llnl/conduit/build-debug/DartConfiguration.tcl -UpdateCTestConfiguration from :/Users/harrison37/Work/github/llnl/conduit/build-debug/DartConfiguration.tcl -Parse Config file:/Users/harrison37/Work/github/llnl/conduit/build-debug/DartConfiguration.tcl -Test project /Users/harrison37/Work/github/llnl/conduit/build-debug -Constructing a list of tests -Done constructing a list of tests -Updating test list for fixtures -Added 0 tests to meet fixture requirements -Checking test dependency graph... -Checking test dependency graph end -test 103 - Start 103: t_conduit_docs_tutorial_python_relay_io_handle_examples - -103: Test command: /Users/harrison37/Work/github/llnl/conduit/uberenv_libs/spack/opt/spack/darwin-highsierra-x86_64/clang-9.0.0-apple/python-2.7.15-3ckhrh3sb7colfuc3gvt6bmkej3b5hlu/bin/python2.7 "-B" "-m" "unittest" "-v" "t_conduit_docs_tutorial_python_relay_io_handle_examples" -103: Environment variables: -103: PYTHONPATH=/Users/harrison37/Work/github/llnl/conduit/build-debug/python-modules/:/Users/harrison37/Work/github/llnl/conduit/src/tests/docs -103: Test timeout computed to be: 1500 -103: test_001_io_handle (t_conduit_docs_tutorial_python_relay_io_handle_examples.Conduit_Tutorial_Python_Relay_IO_Handle) ... ok -103: -103: ---------------------------------------------------------------------- -103: Ran 1 test in 0.063s -103: -103: OK -103: -103: begin: test_001_io_handle,64 -103: -103: Node to write: -103: -103: { -103: "a": -103: { -103: "data": 1.0, -103: "more_data": 2.0, -103: "b": -103: { -103: "my_string": "value" -103: } -103: } -103: } -103: -103: Value at "a/data" = 1.0 -103: -103: Removed "a/more_data" -103: -103: Path "a/more_data" is no more -103: -103: Writing to "a/c" -103: -103: Children of "a": ['data', 'b', 'c'] -103: -103: Read Result: -103: -103: { -103: "a": -103: { -103: "data": 1.0, -103: "b": -103: { -103: "my_string": "value" -103: }, -103: "c": 42.0 -103: } -103: } -103: -103: end: test_001_io_handle,112 -1/1 Test #103: t_conduit_docs_tutorial_python_relay_io_handle_examples ... Passed 0.24 sec - -The following tests passed: - t_conduit_docs_tutorial_python_relay_io_handle_examples - -100% tests passed, 0 tests failed out of 1 - -Total Test time (real) = 0.26 sec diff --git a/src/docs/sphinx/t_conduit_docs_relay_io_generic_examples_out.txt b/src/docs/sphinx/t_conduit_docs_relay_io_generic_examples_out.txt index c163172ac..2933816f5 100644 --- a/src/docs/sphinx/t_conduit_docs_relay_io_generic_examples_out.txt +++ b/src/docs/sphinx/t_conduit_docs_relay_io_generic_examples_out.txt @@ -1,211 +1,137 @@ -Running main() from gtest_main.cc -[==========] Running 6 tests from 1 test case. +Running main() from /Users/harrison37/Work/github/llnl/conduit/src/blt/thirdparty_builtin/googletest-master-2020-01-07/googletest/src/gtest_main.cc +[==========] Running 6 tests from 1 test suite. [----------] Global test environment set-up. [----------] 6 tests from conduit_docs [ RUN ] conduit_docs.relay_io_example_1_json -[/Users/harrison37/Work/github/llnl/conduit/src/tests/docs/t_conduit_docs_relay_io_examples.cpp : 63] - relay_io_example_1_json +BEGIN_EXAMPLE("relay_io_example_1_json") Node to write: -{ - "a": - { - "my_data": 1.0, - "b": - { - "my_string": "value" - } - } -} +a: + my_data: 1.0 + b: + my_string: "value" + Load result: -{ - "a": - { - "my_data": 1.0, - "b": - { - "my_string": "value" - } - } -} -[/Users/harrison37/Work/github/llnl/conduit/src/tests/docs/t_conduit_docs_relay_io_examples.cpp : 81] - relay_io_example_1_json +a: + my_data: 1.0 + b: + my_string: "value" + +END_EXAMPLE("relay_io_example_1_json") [ OK ] conduit_docs.relay_io_example_1_json (1 ms) [ RUN ] conduit_docs.relay_io_example_1_hdf5 -[/Users/harrison37/Work/github/llnl/conduit/src/tests/docs/t_conduit_docs_relay_io_examples.cpp : 90] - relay_io_example_1_hdf5 +BEGIN_EXAMPLE("relay_io_example_1_hdf5") Node to write: -{ - "a": - { - "my_data": 1.0, - "b": - { - "my_string": "value" - } - } -} +a: + my_data: 1.0 + b: + my_string: "value" + Load result: -{ - "a": - { - "my_data": 1.0, - "b": - { - "my_string": "value" - } - } -} -[/Users/harrison37/Work/github/llnl/conduit/src/tests/docs/t_conduit_docs_relay_io_examples.cpp : 108] - relay_io_example_1_hdf5 -[ OK ] conduit_docs.relay_io_example_1_hdf5 (5 ms) +a: + my_data: 1.0 + b: + my_string: "value" + +END_EXAMPLE("relay_io_example_1_hdf5") +[ OK ] conduit_docs.relay_io_example_1_hdf5 (3 ms) [ RUN ] conduit_docs.relay_io_example_2_hdf5 -[/Users/harrison37/Work/github/llnl/conduit/src/tests/docs/t_conduit_docs_relay_io_examples.cpp : 116] - relay_io_example_2_hdf5 +BEGIN_EXAMPLE("relay_io_example_2_hdf5") Node to write: -{ - "a": - { - "my_data": 1.0, - "b": - { - "my_string": "value" - } - } -} +a: + my_data: 1.0 + b: + my_string: "value" + Node to append: -{ - "a": - { - "b": - { - "new_data": 42.0 - } - } -} +a: + b: + new_data: 42.0 + Load result: -{ - "a": - { - "my_data": 1.0, - "b": - { - "my_string": "value", - "new_data": 42.0 - } - } -} -[/Users/harrison37/Work/github/llnl/conduit/src/tests/docs/t_conduit_docs_relay_io_examples.cpp : 141] - relay_io_example_2_hdf5 +a: + my_data: 1.0 + b: + my_string: "value" + new_data: 42.0 + +END_EXAMPLE("relay_io_example_2_hdf5") [ OK ] conduit_docs.relay_io_example_2_hdf5 (3 ms) [ RUN ] conduit_docs.relay_io_example_3_hdf5 -[/Users/harrison37/Work/github/llnl/conduit/src/tests/docs/t_conduit_docs_relay_io_examples.cpp : 148] - relay_io_example_3_hdf5 +BEGIN_EXAMPLE("relay_io_example_3_hdf5") Node to write: -{ - "a": - { - "my_data": 1.0, - "b": - { - "my_string": "value" - } - } -} +a: + my_data: 1.0 + b: + my_string: "value" + Node to load into: -{ - "a": - { - "b": - { - "new_data": 42.0 - } - } -} +a: + b: + new_data: 42.0 + Load result: -{ - "a": - { - "b": - { - "new_data": 42.0, - "my_string": "value" - }, - "my_data": 1.0 - } -} -[/Users/harrison37/Work/github/llnl/conduit/src/tests/docs/t_conduit_docs_relay_io_examples.cpp : 169] - relay_io_example_3_hdf5 -[ OK ] conduit_docs.relay_io_example_3_hdf5 (2 ms) +a: + b: + new_data: 42.0 + my_string: "value" + my_data: 1.0 + +END_EXAMPLE("relay_io_example_3_hdf5") +[ OK ] conduit_docs.relay_io_example_3_hdf5 (1 ms) [ RUN ] conduit_docs.relay_io_example_4_hdf5 -[/Users/harrison37/Work/github/llnl/conduit/src/tests/docs/t_conduit_docs_relay_io_examples.cpp : 177] - relay_io_example_4_hdf5 +BEGIN_EXAMPLE("relay_io_example_4_hdf5") Node to write: -{ - "path": - { - "to": - { - "my_data": 1.0 - } - } -} +path: + to: + my_data: 1.0 + Load result from 'path/to' -{ - "my_data": 1.0 -} -[/Users/harrison37/Work/github/llnl/conduit/src/tests/docs/t_conduit_docs_relay_io_examples.cpp : 194] - relay_io_example_4_hdf5 +my_data: 1.0 + +END_EXAMPLE("relay_io_example_4_hdf5") [ OK ] conduit_docs.relay_io_example_4_hdf5 (2 ms) [ RUN ] conduit_docs.relay_io_example_5_hdf5 -[/Users/harrison37/Work/github/llnl/conduit/src/tests/docs/t_conduit_docs_relay_io_examples.cpp : 201] - relay_io_example_5_hdf5 +BEGIN_EXAMPLE("relay_io_example_5_hdf5") Node to write to 'path/to': -{ - "my_data": 1.0 -} +my_data: 1.0 + Load result: -{ - "path": - { - "to": - { - "my_data": 1.0 - } - } -} -[/Users/harrison37/Work/github/llnl/conduit/src/tests/docs/t_conduit_docs_relay_io_examples.cpp : 218] - relay_io_example_4_hdf5 -[ OK ] conduit_docs.relay_io_example_5_hdf5 (2 ms) -[----------] 6 tests from conduit_docs (16 ms total) +path: + to: + my_data: 1.0 + +END_EXAMPLE("relay_io_example_5_hdf5") +[ OK ] conduit_docs.relay_io_example_5_hdf5 (1 ms) +[----------] 6 tests from conduit_docs (11 ms total) [----------] Global test environment tear-down -[==========] 6 tests from 1 test case ran. (16 ms total) +[==========] 6 tests from 1 test suite ran. (11 ms total) [ PASSED ] 6 tests. diff --git a/src/docs/sphinx/t_conduit_docs_relay_io_handle_examples_out.txt b/src/docs/sphinx/t_conduit_docs_relay_io_handle_examples_out.txt index 58cab3c00..c9b0bb57f 100644 --- a/src/docs/sphinx/t_conduit_docs_relay_io_handle_examples_out.txt +++ b/src/docs/sphinx/t_conduit_docs_relay_io_handle_examples_out.txt @@ -1,24 +1,18 @@ -Running main() from gtest_main.cc -[==========] Running 1 test from 1 test case. +Running main() from /Users/harrison37/Work/github/llnl/conduit/src/blt/thirdparty_builtin/googletest-master-2020-01-07/googletest/src/gtest_main.cc +[==========] Running 1 test from 1 test suite. [----------] Global test environment set-up. [----------] 1 test from conduit_docs [ RUN ] conduit_docs.relay_io_handle_1 -[/Users/harrison37/Work/github/llnl/conduit/src/tests/docs/t_conduit_docs_relay_io_handle_examples.cpp : 63] - relay_io_handle_example_1 +BEGIN_EXAMPLE("relay_io_handle_example_1") Node to write: -{ - "a": - { - "data": 1.0, - "more_data": 2.0, - "b": - { - "my_string": "value" - } - } -} +a: + data: 1.0 + more_data: 2.0 + b: + my_string: "value" + Value at "a/data" = 1 @@ -32,22 +26,16 @@ Children of "a": "data" "b" "c" Read Result: -{ - "a": - { - "data": 1.0, - "b": - { - "my_string": "value" - }, - "c": 42.0 - } -} -[/Users/harrison37/Work/github/llnl/conduit/src/tests/docs/t_conduit_docs_relay_io_handle_examples.cpp : 134] - relay_io_handle_example_1 -[ OK ] conduit_docs.relay_io_handle_1 (7 ms) -[----------] 1 test from conduit_docs (7 ms total) +a: + data: 1.0 + b: + my_string: "value" + c: 42.0 + +END_EXAMPLE("relay_io_handle_example_1") +[ OK ] conduit_docs.relay_io_handle_1 (5 ms) +[----------] 1 test from conduit_docs (5 ms total) [----------] Global test environment tear-down -[==========] 1 test from 1 test case ran. (8 ms total) +[==========] 1 test from 1 test suite ran. (5 ms total) [ PASSED ] 1 test. diff --git a/src/docs/sphinx/t_conduit_docs_relay_io_hdf5_examples_out.txt b/src/docs/sphinx/t_conduit_docs_relay_io_hdf5_examples_out.txt index 8c78f9f6e..cefd7b344 100644 --- a/src/docs/sphinx/t_conduit_docs_relay_io_hdf5_examples_out.txt +++ b/src/docs/sphinx/t_conduit_docs_relay_io_hdf5_examples_out.txt @@ -1,23 +1,17 @@ -Running main() from gtest_main.cc -[==========] Running 2 tests from 1 test case. +Running main() from /Users/harrison37/Work/github/llnl/conduit/src/blt/thirdparty_builtin/googletest-master-2020-01-07/googletest/src/gtest_main.cc +[==========] Running 2 tests from 1 test suite. [----------] Global test environment set-up. [----------] 2 tests from conduit_docs [ RUN ] conduit_docs.relay_io_example_hdf5_interface_1 -[/Users/harrison37/Work/github/llnl/conduit/src/tests/docs/t_conduit_docs_relay_io_hdf5_examples.cpp : 64] - relay_io_example_hdf5_interface_1 +BEGIN_EXAMPLE("relay_io_example_hdf5_interface_1") Node to write: -{ - "a": - { - "my_data": 1.0, - "b": - { - "my_string": "value" - } - } -} +a: + my_data: 1.0 + b: + my_string: "value" + Path 'myoutput.hdf5:a/my_data' exists @@ -26,90 +20,64 @@ Data loaded: Data loaded: -{ - "a": - { - "my_data": 3.1415, - "b": - { - "my_string": "value", - "c": 144 - } - } -} +a: + my_data: 3.1415 + b: + my_string: "value" + c: 144 + File 'myoutput.hdf5' is a hdf5 file -[/Users/harrison37/Work/github/llnl/conduit/src/tests/docs/t_conduit_docs_relay_io_hdf5_examples.cpp : 116] - relay_io_example_hdf5_interface_1 -[ OK ] conduit_docs.relay_io_example_hdf5_interface_1 (5 ms) +END_EXAMPLE("relay_io_example_hdf5_interface_1") +[ OK ] conduit_docs.relay_io_example_hdf5_interface_1 (4 ms) [ RUN ] conduit_docs.relay_io_example_hdf5_interface_2 -[/Users/harrison37/Work/github/llnl/conduit/src/tests/docs/t_conduit_docs_relay_io_hdf5_examples.cpp : 126] - relay_io_example_hdf5_interface_opts +BEGIN_EXAMPLE("relay_io_example_hdf5_interface_opts") Relay I/O Info and Default Options: -{ - "protocols": - { - "json": "enabled", - "conduit_json": "enabled", - "conduit_bin": "enabled", - "hdf5": "enabled", - "conduit_silo": "disabled", - "conduit_silo_mesh": "disabled", - "adios": "disabled" - }, - "options": - { - "hdf5": - { - "compact_storage": - { - "enabled": "true", - "threshold": 1024 - }, - "chunking": - { - "enabled": "true", - "threshold": 2000000, - "chunk_size": 1000000, - "compression": - { - "method": "gzip", - "level": 5 - } - } - } - } -} +protocols: + json: "enabled" + conduit_json: "enabled" + conduit_base64_json: "enabled" + yaml: "enabled" + conduit_bin: "enabled" + hdf5: "enabled" + conduit_silo: "disabled" + conduit_silo_mesh: "disabled" + adios: "disabled" +options: + hdf5: + compact_storage: + enabled: "true" + threshold: 1024 + chunking: + enabled: "true" + threshold: 2000000 + chunk_size: 1000000 + compression: + method: "gzip" + level: 5 + New HDF5 I/O Options: -{ - "compact_storage": - { - "enabled": "true", - "threshold": 1024 - }, - "chunking": - { - "enabled": "true", - "threshold": 2000, - "chunk_size": 2000, - "compression": - { - "method": "gzip", - "level": 5 - } - } -} +compact_storage: + enabled: "true" + threshold: 1024 +chunking: + enabled: "true" + threshold: 2000 + chunk_size: 2000 + compression: + method: "gzip" + level: 5 + saving data to 'myoutput_chunked.hdf5' -[/Users/harrison37/Work/github/llnl/conduit/src/tests/docs/t_conduit_docs_relay_io_hdf5_examples.cpp : 161] - relay_io_example_hdf5_interface_opts -[ OK ] conduit_docs.relay_io_example_hdf5_interface_2 (2 ms) +END_EXAMPLE("relay_io_example_hdf5_interface_opts") +[ OK ] conduit_docs.relay_io_example_hdf5_interface_2 (3 ms) [----------] 2 tests from conduit_docs (7 ms total) [----------] Global test environment tear-down -[==========] 2 tests from 1 test case ran. (7 ms total) +[==========] 2 tests from 1 test suite ran. (7 ms total) [ PASSED ] 2 tests. diff --git a/src/docs/sphinx/t_conduit_docs_tutorial_basics_out.txt b/src/docs/sphinx/t_conduit_docs_tutorial_basics_out.txt index 268a3f513..b8195da09 100644 --- a/src/docs/sphinx/t_conduit_docs_tutorial_basics_out.txt +++ b/src/docs/sphinx/t_conduit_docs_tutorial_basics_out.txt @@ -1,150 +1,111 @@ -Running main() from gtest_main.cc -[==========] Running 7 tests from 1 test case. +Running main() from /Users/harrison37/Work/github/llnl/conduit/src/blt/thirdparty_builtin/googletest-master-2020-01-07/googletest/src/gtest_main.cc +[==========] Running 7 tests from 1 test suite. [----------] Global test environment set-up. [----------] 7 tests from conduit_tutorial [ RUN ] conduit_tutorial.basics_very_basic -[/Users/harrison37/Work/conduit/src/tests/docs/t_conduit_docs_tutorial_basics.cpp : 63] - basics_very_basic +BEGIN_EXAMPLE("basics_very_basic") -{ - "my": "data" -} -[/Users/harrison37/Work/conduit/src/tests/docs/t_conduit_docs_tutorial_basics.cpp : 69] - basics_very_basic +my: "data" + +END_EXAMPLE("basics_very_basic") [ OK ] conduit_tutorial.basics_very_basic (0 ms) [ RUN ] conduit_tutorial.basics_hierarchial -[/Users/harrison37/Work/conduit/src/tests/docs/t_conduit_docs_tutorial_basics.cpp : 75] - basics_hierarchial +BEGIN_EXAMPLE("basics_hierarchial") + +my: "data" +a: + b: + c: "d" + e: 64.0 -{ - "my": "data", - "a": - { - "b": - { - "c": "d", - "e": 64.0 - } - } -} total bytes: 15 -[/Users/harrison37/Work/conduit/src/tests/docs/t_conduit_docs_tutorial_basics.cpp : 85] - basics_hierarchial +END_EXAMPLE("basics_hierarchial") [ OK ] conduit_tutorial.basics_hierarchial (0 ms) [ RUN ] conduit_tutorial.basics_object_and_list -[/Users/harrison37/Work/conduit/src/tests/docs/t_conduit_docs_tutorial_basics.cpp : 91] - basics_object_and_list +BEGIN_EXAMPLE("basics_object_and_list") -{ - "object_example": - { - "val1": "data", - "val2": 10, - "val3": 3.1415 - }, - "list_example": - [ - 0, - 1, - 2, - 3, - 4 - ] -} -[/Users/harrison37/Work/conduit/src/tests/docs/t_conduit_docs_tutorial_basics.cpp : 106] - basics_object_and_list +object_example: + val1: "data" + val2: 10 + val3: 3.1415 +list_example: + - 0 + - 1 + - 2 + - 3 + - 4 + +END_EXAMPLE("basics_object_and_list") [ OK ] conduit_tutorial.basics_object_and_list (0 ms) [ RUN ] conduit_tutorial.basics_object_and_list_itr -[/Users/harrison37/Work/conduit/src/tests/docs/t_conduit_docs_tutorial_basics.cpp : 112] - basics_object_and_list_itr +BEGIN_EXAMPLE("basics_object_and_list_itr") + +object_example: + val1: "data" + val2: 10 + val3: 3.1415 +list_example: + - 0 + - 1 + - 2 + - 3 + - 4 -{ - "object_example": - { - "val1": "data", - "val2": 10, - "val3": 3.1415 - }, - "list_example": - [ - 0, - 1, - 2, - 3, - 4 - ] -} val1: "data" val2: 10 val3: 3.1415 + 0 1 2 3 4 -[/Users/harrison37/Work/conduit/src/tests/docs/t_conduit_docs_tutorial_basics.cpp : 143] - basics_object_and_list_itr +END_EXAMPLE("basics_object_and_list_itr") [ OK ] conduit_tutorial.basics_object_and_list_itr (0 ms) [ RUN ] conduit_tutorial.basics_mem_spaces -[/Users/harrison37/Work/conduit/src/tests/docs/t_conduit_docs_tutorial_basics.cpp : 150] - basics_mem_spaces +BEGIN_EXAMPLE("basics_mem_spaces") -{ - "mem_spaces": - { - "0x7fcc834044e0": - { - "path": "my", - "type": "allocated", - "bytes": 5 - }, - "0x7fcc83405f20": - { - "path": "a/b/c", - "type": "allocated", - "bytes": 2 - }, - "0x7fcc83405f10": - { - "path": "a/b/e", - "type": "allocated", - "bytes": 8 - } - }, - "total_bytes_allocated": 15, - "total_bytes_mmaped": 0, - "total_bytes_compact": 15, - "total_strided_bytes": 15 -} -[/Users/harrison37/Work/conduit/src/tests/docs/t_conduit_docs_tutorial_basics.cpp : 161] - basics_mem_spaces +mem_spaces: + 0x7f9239c04e40: + path: "my" + type: "allocated" + bytes: 5 + 0x7f9239c05420: + path: "a/b/c" + type: "allocated" + bytes: 2 + 0x7f9239c05e50: + path: "a/b/e" + type: "allocated" + bytes: 8 +total_bytes_allocated: 15 +total_bytes_mmaped: 0 +total_bytes_compact: 15 +total_strided_bytes: 15 + +END_EXAMPLE("basics_mem_spaces") [ OK ] conduit_tutorial.basics_mem_spaces (0 ms) [ RUN ] conduit_tutorial.basics_bw_style -[/Users/harrison37/Work/conduit/src/tests/docs/t_conduit_docs_tutorial_basics.cpp : 167] - basics_bw_style +BEGIN_EXAMPLE("basics_bw_style") + +test: 100 -{ - "test": 100 -} { "test": {"dtype":"uint32", "number_of_elements": 1, "offset": 0, "stride": 4, "element_bytes": 4, "endianness": "little", "value": 100} } -[/Users/harrison37/Work/conduit/src/tests/docs/t_conduit_docs_tutorial_basics.cpp : 175] - basics_bw_style +END_EXAMPLE("basics_bw_style") [ OK ] conduit_tutorial.basics_bw_style (0 ms) [ RUN ] conduit_tutorial.basics_bw_style_from_native -[/Users/harrison37/Work/conduit/src/tests/docs/t_conduit_docs_tutorial_basics.cpp : 181] - basics_bw_style_from_native +BEGIN_EXAMPLE("basics_bw_style_from_native") { "test": {"dtype":"int32", "number_of_elements": 1, "offset": 0, "stride": 4, "element_bytes": 4, "endianness": "little", "value": 100} } -[/Users/harrison37/Work/conduit/src/tests/docs/t_conduit_docs_tutorial_basics.cpp : 188] - basics_bw_style_from_native -[ OK ] conduit_tutorial.basics_bw_style_from_native (1 ms) -[----------] 7 tests from conduit_tutorial (1 ms total) +END_EXAMPLE("basics_bw_style_from_native") +[ OK ] conduit_tutorial.basics_bw_style_from_native (0 ms) +[----------] 7 tests from conduit_tutorial (0 ms total) [----------] Global test environment tear-down -[==========] 7 tests from 1 test case ran. (1 ms total) +[==========] 7 tests from 1 test suite ran. (1 ms total) [ PASSED ] 7 tests. diff --git a/src/docs/sphinx/t_conduit_docs_tutorial_errors_out.txt b/src/docs/sphinx/t_conduit_docs_tutorial_errors_out.txt index 8c3a4a2bb..4b5b491f0 100644 --- a/src/docs/sphinx/t_conduit_docs_tutorial_errors_out.txt +++ b/src/docs/sphinx/t_conduit_docs_tutorial_errors_out.txt @@ -1,26 +1,24 @@ -Running main() from gtest_main.cc -[==========] Running 3 tests from 1 test case. +Running main() from /Users/harrison37/Work/github/llnl/conduit/src/blt/thirdparty_builtin/googletest-master-2020-01-07/googletest/src/gtest_main.cc +[==========] Running 3 tests from 1 test suite. [----------] Global test environment set-up. [----------] 3 tests from conduit_tutorial [ RUN ] conduit_tutorial.error_handlers_funcs_start -[/Users/harrison37/Work/conduit/src/tests/docs/t_conduit_docs_tutorial_errors.cpp : 63] - error_handlers_funcs +BEGIN_EXAMPLE("error_handlers_funcs") [ OK ] conduit_tutorial.error_handlers_funcs_start (0 ms) [ RUN ] conduit_tutorial.error_handlers_funcs_end -[/Users/harrison37/Work/conduit/src/tests/docs/t_conduit_docs_tutorial_errors.cpp : 94] - error_handlers_funcs +END_EXAMPLE("error_handlers_funcs") [ OK ] conduit_tutorial.error_handlers_funcs_end (0 ms) [ RUN ] conduit_tutorial.error_handlers -[/Users/harrison37/Work/conduit/src/tests/docs/t_conduit_docs_tutorial_errors.cpp : 101] - error_handlers +BEGIN_EXAMPLE("error_handlers_rewire") [INFO] An info message [WARNING!] Node::as_float32() const -- DataType float64 at path my_value does not equal expected DataType float32 -[ERROR!] Cannot const fetch_existing, Node(my_value) is not an object -[/Users/harrison37/Work/conduit/src/tests/docs/t_conduit_docs_tutorial_errors.cpp : 137] - error_handlers -[ OK ] conduit_tutorial.error_handlers (1 ms) -[----------] 3 tests from conduit_tutorial (1 ms total) +[ERROR!] Cannot fetch_existing, Node(my_value) is not an object +END_EXAMPLE("error_handlers_rewire") +BEGIN_EXAMPLE("error_handlers_reset") +END_EXAMPLE("error_handlers_reset") +[ OK ] conduit_tutorial.error_handlers (0 ms) +[----------] 3 tests from conduit_tutorial (0 ms total) [----------] Global test environment tear-down -[==========] 3 tests from 1 test case ran. (1 ms total) +[==========] 3 tests from 1 test suite ran. (0 ms total) [ PASSED ] 3 tests. diff --git a/src/docs/sphinx/t_conduit_docs_tutorial_numeric_out.txt b/src/docs/sphinx/t_conduit_docs_tutorial_numeric_out.txt index 737f5a027..42be0963e 100644 --- a/src/docs/sphinx/t_conduit_docs_tutorial_numeric_out.txt +++ b/src/docs/sphinx/t_conduit_docs_tutorial_numeric_out.txt @@ -1,67 +1,45 @@ -Running main() from gtest_main.cc -[==========] Running 8 tests from 1 test case. +Running main() from /Users/harrison37/Work/github/llnl/conduit/src/blt/thirdparty_builtin/googletest-master-2020-01-07/googletest/src/gtest_main.cc +[==========] Running 8 tests from 1 test suite. [----------] Global test environment set-up. [----------] 8 tests from conduit_tutorial [ RUN ] conduit_tutorial.numeric_as_dtype -[/Users/harrison37/Work/conduit/src/tests/docs/t_conduit_docs_tutorial_numeric.cpp : 63] - numeric_as_dtype -# _conduit_tutorial_cpp_numeric_node_as_dtype_out_start +BEGIN_EXAMPLE("numeric_as_dtype") 100 -# _conduit_tutorial_cpp_numeric_node_as_dtype_out_end -[/Users/harrison37/Work/conduit/src/tests/docs/t_conduit_docs_tutorial_numeric.cpp : 70] - numeric_as_dtype +END_EXAMPLE("numeric_as_dtype") [ OK ] conduit_tutorial.numeric_as_dtype (0 ms) [ RUN ] conduit_tutorial.numeric_via_value -[/Users/harrison37/Work/conduit/src/tests/docs/t_conduit_docs_tutorial_numeric.cpp : 76] - numeric_via_value -# _conduit_tutorial_cpp_numeric_node_value_out_start +BEGIN_EXAMPLE("numeric_via_value") 100 -# _conduit_tutorial_cpp_numeric_node_value_out_end -[/Users/harrison37/Work/conduit/src/tests/docs/t_conduit_docs_tutorial_numeric.cpp : 84] - numeric_via_value +END_EXAMPLE("numeric_via_value") [ OK ] conduit_tutorial.numeric_via_value (0 ms) [ RUN ] conduit_tutorial.numeric_ptr_as_dtype -[/Users/harrison37/Work/conduit/src/tests/docs/t_conduit_docs_tutorial_numeric.cpp : 91] - numeric_ptr_as_dtype -# _conduit_tutorial_cpp_numeric_array_node_pointer_out_end +BEGIN_EXAMPLE("numeric_ptr_as_dtype") my_vals[0] = 100 my_vals[1] = 200 my_vals[2] = 300 my_vals[3] = 400 -# _conduit_tutorial_cpp_numeric_array_node_pointer_out_end -[/Users/harrison37/Work/conduit/src/tests/docs/t_conduit_docs_tutorial_numeric.cpp : 105] - numeric_ptr_as_dtype +END_EXAMPLE("numeric_ptr_as_dtype") [ OK ] conduit_tutorial.numeric_ptr_as_dtype (0 ms) [ RUN ] conduit_tutorial.numeric_ptr_via_value -[/Users/harrison37/Work/conduit/src/tests/docs/t_conduit_docs_tutorial_numeric.cpp : 111] - numeric_ptr_via_value -# _conduit_tutorial_cpp_numeric_array_node_value_out_start +BEGIN_EXAMPLE("numeric_ptr_via_value") my_vals[0] = 100 my_vals[1] = 200 my_vals[2] = 300 my_vals[3] = 400 -# _conduit_tutorial_cpp_numeric_array_node_value_out_end -[/Users/harrison37/Work/conduit/src/tests/docs/t_conduit_docs_tutorial_numeric.cpp : 125] - numeric_ptr_via_value +END_EXAMPLE("numeric_ptr_via_value") [ OK ] conduit_tutorial.numeric_ptr_via_value (0 ms) [ RUN ] conduit_tutorial.numeric_strided_data_array -[/Users/harrison37/Work/conduit/src/tests/docs/t_conduit_docs_tutorial_numeric.cpp : 131] - numeric_strided_data_array -# _conduit_tutorial_cpp_numeric_non-contiguous_out_start +BEGIN_EXAMPLE("numeric_strided_data_array") my_vals[0] = 100 my_vals[1] = 300 [100, 300] -# _conduit_tutorial_cpp_numeric_non-contiguous_out_end -[/Users/harrison37/Work/conduit/src/tests/docs/t_conduit_docs_tutorial_numeric.cpp : 149] - numeric_strided_data_array +END_EXAMPLE("numeric_strided_data_array") [ OK ] conduit_tutorial.numeric_strided_data_array (0 ms) [ RUN ] conduit_tutorial.numeric_double_conversion_start -[/Users/harrison37/Work/conduit/src/tests/docs/t_conduit_docs_tutorial_numeric.cpp : 155] - numeric_double_conversion_start +BEGIN_EXAMPLE("numeric_double_conversion") [ OK ] conduit_tutorial.numeric_double_conversion_start (0 ms) [ RUN ] conduit_tutorial.numeric_double_conversion float32 case: -# _conduit_tutorial_cpp_numeric_introspection_out_start converting to temporary double array [100.0, 200.0, 300.0, 400.0] vals[0] = 100 @@ -75,14 +53,12 @@ vals[0] = 1000 vals[1] = 2000 vals[2] = 3000 vals[3] = 4000 -# _conduit_tutorial_cpp_numeric_introspection_out_end -[ OK ] conduit_tutorial.numeric_double_conversion (0 ms) +[ OK ] conduit_tutorial.numeric_double_conversion (1 ms) [ RUN ] conduit_tutorial.numeric_double_conversion_end -[/Users/harrison37/Work/conduit/src/tests/docs/t_conduit_docs_tutorial_numeric.cpp : 221] - numeric_double_conversion_end -[ OK ] conduit_tutorial.numeric_double_conversion_end (1 ms) +END_EXAMPLE("numeric_double_conversion") +[ OK ] conduit_tutorial.numeric_double_conversion_end (0 ms) [----------] 8 tests from conduit_tutorial (1 ms total) [----------] Global test environment tear-down -[==========] 8 tests from 1 test case ran. (1 ms total) +[==========] 8 tests from 1 test suite ran. (1 ms total) [ PASSED ] 8 tests. diff --git a/src/docs/sphinx/t_conduit_docs_tutorial_ownership_out.txt b/src/docs/sphinx/t_conduit_docs_tutorial_ownership_out.txt index 6a6d01bae..aaad41d38 100644 --- a/src/docs/sphinx/t_conduit_docs_tutorial_ownership_out.txt +++ b/src/docs/sphinx/t_conduit_docs_tutorial_ownership_out.txt @@ -1,46 +1,35 @@ -Running main() from gtest_main.cc -[==========] Running 1 test from 1 test case. +Running main() from /Users/harrison37/Work/github/llnl/conduit/src/blt/thirdparty_builtin/googletest-master-2020-01-07/googletest/src/gtest_main.cc +[==========] Running 1 test from 1 test suite. [----------] Global test environment set-up. [----------] 1 test from conduit_tutorial [ RUN ] conduit_tutorial.mem_ownership_external -[/Users/harrison37/Work/conduit/src/tests/docs/t_conduit_docs_tutorial_ownership.cpp : 63] - mem_ownership_external +BEGIN_EXAMPLE("mem_ownership_external") -{ - "mem_spaces": - { - "0x7fdebac044d0": - { - "path": "v_owned", - "type": "allocated", - "bytes": 40 - }, - "0x7fdebac045a0": - { - "path": "v_external", - "type": "external" - } - }, - "total_bytes_allocated": 40, - "total_bytes_mmaped": 0, - "total_bytes_compact": 80, - "total_strided_bytes": 80 -} +mem_spaces: + 0x7fd547c04c50: + path: "v_owned" + type: "allocated" + bytes: 40 + 0x7fd547c04890: + path: "v_external" + type: "external" +total_bytes_allocated: 40 +total_bytes_mmaped: 0 +total_bytes_compact: 80 +total_strided_bytes: 80 -{ - "v_owned": [0.0, 3.1415, 6.283, 9.4245, 12.566], - "v_external": [0.0, 3.1415, 6.283, 9.4245, 12.566] -} -{ - "v_owned": [0.0, 3.1415, 6.283, 9.4245, 12.566], - "v_external": [0.0, -3.1415, 6.283, 9.4245, 12.566] -} -[/Users/harrison37/Work/conduit/src/tests/docs/t_conduit_docs_tutorial_ownership.cpp : 83] - mem_ownership_external -[ OK ] conduit_tutorial.mem_ownership_external (0 ms) -[----------] 1 test from conduit_tutorial (0 ms total) +v_owned: [0.0, 3.1415, 6.283, 9.4245, 12.566] +v_external: [0.0, 3.1415, 6.283, 9.4245, 12.566] + + +v_owned: [0.0, 3.1415, 6.283, 9.4245, 12.566] +v_external: [0.0, -3.1415, 6.283, 9.4245, 12.566] + +END_EXAMPLE("mem_ownership_external") +[ OK ] conduit_tutorial.mem_ownership_external (1 ms) +[----------] 1 test from conduit_tutorial (1 ms total) [----------] Global test environment tear-down -[==========] 1 test from 1 test case ran. (0 ms total) +[==========] 1 test from 1 test suite ran. (1 ms total) [ PASSED ] 1 test. diff --git a/src/docs/sphinx/t_conduit_docs_tutorial_parse_out.txt b/src/docs/sphinx/t_conduit_docs_tutorial_parse_out.txt index d92517d59..4f7219c34 100644 --- a/src/docs/sphinx/t_conduit_docs_tutorial_parse_out.txt +++ b/src/docs/sphinx/t_conduit_docs_tutorial_parse_out.txt @@ -1,5 +1,5 @@ -Running main() from gtest_main.cc -[==========] Running 8 tests from 1 test case. +Running main() from /Users/harrison37/Work/github/llnl/conduit/src/blt/thirdparty_builtin/googletest-master-2020-01-07/googletest/src/gtest_main.cc +[==========] Running 8 tests from 1 test suite. [----------] Global test environment set-up. [----------] 8 tests from conduit_tutorial [ RUN ] conduit_tutorial.t_conduit_docs_tutorial_yaml @@ -10,7 +10,7 @@ BEGIN_EXAMPLE("t_conduit_docs_tutorial_yaml") "mykey": {"dtype":"float64", "number_of_elements": 1, "offset": 0, "stride": 8, "element_bytes": 8, "endianness": "little", "value": 42.0} } END_EXAMPLE("t_conduit_docs_tutorial_yaml") -[ OK ] conduit_tutorial.t_conduit_docs_tutorial_yaml (1 ms) +[ OK ] conduit_tutorial.t_conduit_docs_tutorial_yaml (0 ms) [ RUN ] conduit_tutorial.t_conduit_docs_tutorial_json BEGIN_EXAMPLE("t_conduit_docs_tutorial_json") 42 @@ -33,9 +33,8 @@ END_EXAMPLE("t_conduit_docs_tutorial_yaml_inline_array") BEGIN_EXAMPLE("t_json_generator_std") 100 -{ - "test": 100.0 -} +test: 100.0 + { "test": {"dtype":"float64", "number_of_elements": 1, "offset": 0, "stride": 8, "element_bytes": 8, "endianness": "little", "value": 100.0} @@ -50,9 +49,8 @@ BEGIN_EXAMPLE("t_json_generator_pure_json") "test": {"dtype":"float64", "number_of_elements": 1, "offset": 0, "stride": 8, "element_bytes": 8, "endianness": "little", "value": 100.0} } -{ - "test": 100.0 -} +test: 100.0 + END_EXAMPLE("t_json_generator_pure_json") [ OK ] conduit_tutorial.json_generator_pure_json (0 ms) [ RUN ] conduit_tutorial.json_generator_pure_yaml @@ -63,9 +61,8 @@ BEGIN_EXAMPLE("t_json_generator_pure_yaml") "test": {"dtype":"float64", "number_of_elements": 1, "offset": 0, "stride": 8, "element_bytes": 8, "endianness": "little", "value": 100.0} } -{ - "test": 100.0 -} +test: 100.0 + END_EXAMPLE("t_json_generator_pure_yaml") [ OK ] conduit_tutorial.json_generator_pure_yaml (0 ms) [ RUN ] conduit_tutorial.json_generator_bind_to_incore @@ -73,100 +70,74 @@ BEGIN_EXAMPLE("t_json_generator_bind_to_incore") 100 vs 100 200 vs 200 -{ - "a": 100.0, - "b": 200.0 -} +a: 100.0 +b: 200.0 + + +mem_spaces: + 0x7ffeea50a030: + path: "a" + type: "external" +total_bytes_allocated: 0 +total_bytes_mmaped: 0 +total_bytes_compact: 16 +total_strided_bytes: 16 -{ - "mem_spaces": - { - "0x7ffee1aafda0": - { - "path": "a", - "type": "external" - } - }, - "total_bytes_allocated": 0, - "total_bytes_mmaped": 0, - "total_bytes_compact": 16, - "total_strided_bytes": 16 -} END_EXAMPLE("t_json_generator_bind_to_incore") -[ OK ] conduit_tutorial.json_generator_bind_to_incore (0 ms) +[ OK ] conduit_tutorial.json_generator_bind_to_incore (1 ms) [ RUN ] conduit_tutorial.json_generator_compact BEGIN_EXAMPLE("t_json_generator_compact") [100.0, 200.0, 300.0, 400.0, 500.0] [-100.0, -200.0, -300.0, -400.0, -500.0] -{ - "mem_spaces": - { - "0x7ffee1aafd60": - { - "path": "", - "type": "external" - } - }, - "total_bytes_allocated": 0, - "total_bytes_mmaped": 0, - "total_bytes_compact": 40, - "total_strided_bytes": 72 -} +mem_spaces: + 0x7ffeea509ff0: + path: "" + type: "external" +total_bytes_allocated: 0 +total_bytes_mmaped: 0 +total_bytes_compact: 40 +total_strided_bytes: 72 + + +mem_spaces: + 0x7ffeea509ff0: + path: "" + type: "external" +total_bytes_allocated: 0 +total_bytes_mmaped: 0 +total_bytes_compact: 40 +total_strided_bytes: 72 -{ - "mem_spaces": - { - "0x7ffee1aafd60": - { - "path": "", - "type": "external" - } - }, - "total_bytes_allocated": 0, - "total_bytes_mmaped": 0, - "total_bytes_compact": 40, - "total_strided_bytes": 72 -} [100.0, 200.0, 300.0, 400.0, 500.0] {"dtype":"float64", "number_of_elements": 5, "offset": 0, "stride": 8, "element_bytes": 8, "endianness": "little"} -{ - "mem_spaces": - { - "0x7f8324700680": - { - "path": "", - "type": "allocated", - "bytes": 40 - } - }, - "total_bytes_allocated": 40, - "total_bytes_mmaped": 0, - "total_bytes_compact": 40, - "total_strided_bytes": 40 -} +mem_spaces: + 0x7fa1db4056e0: + path: "" + type: "allocated" + bytes: 40 +total_bytes_allocated: 40 +total_bytes_mmaped: 0 +total_bytes_compact: 40 +total_strided_bytes: 40 + [-100.0, -200.0, -300.0, -400.0, -500.0] -{ - "mem_spaces": - { - "0x7f8324700600": - { - "path": "", - "type": "allocated", - "bytes": 40 - } - }, - "total_bytes_allocated": 40, - "total_bytes_mmaped": 0, - "total_bytes_compact": 40, - "total_strided_bytes": 40 -} +mem_spaces: + 0x7fa1db406350: + path: "" + type: "allocated" + bytes: 40 +total_bytes_allocated: 40 +total_bytes_mmaped: 0 +total_bytes_compact: 40 +total_strided_bytes: 40 + END_EXAMPLE("t_json_generator_compact") -[ OK ] conduit_tutorial.json_generator_compact (1 ms) -[----------] 8 tests from conduit_tutorial (2 ms total) +[ OK ] conduit_tutorial.json_generator_compact (0 ms) +[----------] 8 tests from conduit_tutorial (1 ms total) [----------] Global test environment tear-down -[==========] 8 tests from 1 test case ran. (2 ms total) +[==========] 8 tests from 1 test suite ran. (1 ms total) [ PASSED ] 8 tests. diff --git a/src/docs/sphinx/t_conduit_docs_tutorial_python_basics_out.txt b/src/docs/sphinx/t_conduit_docs_tutorial_python_basics_out.txt new file mode 100644 index 000000000..3d3574669 --- /dev/null +++ b/src/docs/sphinx/t_conduit_docs_tutorial_python_basics_out.txt @@ -0,0 +1,123 @@ +UpdateCTestConfiguration from :/Users/harrison37/Work/github/llnl/conduit/build-debug/DartConfiguration.tcl +Parse Config file:/Users/harrison37/Work/github/llnl/conduit/build-debug/DartConfiguration.tcl +UpdateCTestConfiguration from :/Users/harrison37/Work/github/llnl/conduit/build-debug/DartConfiguration.tcl +Parse Config file:/Users/harrison37/Work/github/llnl/conduit/build-debug/DartConfiguration.tcl +Test project /Users/harrison37/Work/github/llnl/conduit/build-debug +Constructing a list of tests +Done constructing a list of tests +Updating test list for fixtures +Added 0 tests to meet fixture requirements +Checking test dependency graph... +Checking test dependency graph end +test 106 + Start 106: t_conduit_docs_tutorial_python_basics + +106: Test command: /Users/harrison37/Work/github/llnl/conduit/uberenv_libs/spack/opt/spack/darwin-mojave-x86_64/clang-10.0.0-apple/python-3.7.4-tuamu4qn6msg6bqa5hd7glbv5uxm5lup/bin/python3.7 "-B" "-m" "unittest" "-v" "t_conduit_docs_tutorial_python_basics" +106: Environment variables: +106: PYTHONPATH=/Users/harrison37/Work/github/llnl/conduit/build-debug/python-modules/:/Users/harrison37/Work/github/llnl/conduit/src/tests/docs +106: Test timeout computed to be: 1500 +106: test_001_basics_very_basic (t_conduit_docs_tutorial_python_basics.Conduit_Tutorial_Python_Basics) ... ok +106: test_002_basics_hierarchial (t_conduit_docs_tutorial_python_basics.Conduit_Tutorial_Python_Basics) ... ok +106: test_003_basics_object_and_list (t_conduit_docs_tutorial_python_basics.Conduit_Tutorial_Python_Basics) ... ok +106: test_004_basics_object_and_list_itr (t_conduit_docs_tutorial_python_basics.Conduit_Tutorial_Python_Basics) ... ok +106: test_005_basics_mem_spaces (t_conduit_docs_tutorial_python_basics.Conduit_Tutorial_Python_Basics) ... ok +106: test_006_basics_bw_style (t_conduit_docs_tutorial_python_basics.Conduit_Tutorial_Python_Basics) ... ok +106: test_007_basics_bw_style_from_native (t_conduit_docs_tutorial_python_basics.Conduit_Tutorial_Python_Basics) ... ok +106: +106: ---------------------------------------------------------------------- +106: Ran 7 tests in 0.001s +106: +106: OK +106: BEGIN_EXAMPLE("py_basics_very_basic") +106: +106: my: "data" +106: +106: END_EXAMPLE("py_basics_very_basic") +106: BEGIN_EXAMPLE("py_basics_hierarchial") +106: +106: my: "data" +106: a: +106: b: +106: c: "d" +106: e: 64.0 +106: +106: total bytes: 15 +106: +106: END_EXAMPLE("py_basics_hierarchial") +106: BEGIN_EXAMPLE("py_basics_object_and_list") +106: +106: object_example: +106: val1: "data" +106: val2: 10 +106: val3: 3.1415 +106: list_example: +106: - 0 +106: - 1 +106: - 2 +106: - 3 +106: - 4 +106: +106: END_EXAMPLE("py_basics_object_and_list") +106: BEGIN_EXAMPLE("py_basics_object_and_list_itr") +106: +106: object_example: +106: val1: "data" +106: val2: 10 +106: val3: 3.1415 +106: list_example: +106: - 0 +106: - 1 +106: - 2 +106: - 3 +106: - 4 +106: +106: val1: "data" +106: val2: 10 +106: val3: 3.1415 +106: +106: 0 +106: 1 +106: 2 +106: 3 +106: 4 +106: END_EXAMPLE("py_basics_object_and_list_itr") +106: BEGIN_EXAMPLE("py_basics_mem_spaces") +106: +106: mem_spaces: +106: 0x7ff41ae58e90: +106: path: "my" +106: type: "allocated" +106: bytes: 5 +106: 0x7ff41ae84120: +106: path: "a/b/c" +106: type: "allocated" +106: bytes: 2 +106: 0x7ff41ae7d8a0: +106: path: "a/b/e" +106: type: "allocated" +106: bytes: 8 +106: total_bytes_allocated: 15 +106: total_bytes_mmaped: 0 +106: total_bytes_compact: 15 +106: total_strided_bytes: 15 +106: +106: END_EXAMPLE("py_basics_mem_spaces") +106: BEGIN_EXAMPLE("py_basics_bw_style") +106: +106: test: 100 +106: +106: END_EXAMPLE("py_basics_bw_style") +106: BEGIN_EXAMPLE("py_basics_bw_style_from_native") +106: +106: { +106: "test": {"dtype":"int64", "number_of_elements": 1, "offset": 0, "stride": 8, "element_bytes": 8, "endianness": "little"} +106: } +106: END_EXAMPLE("py_basics_bw_style_from_native") +1/1 Test #106: t_conduit_docs_tutorial_python_basics ... Passed 0.15 sec + +The following tests passed: + t_conduit_docs_tutorial_python_basics + +100% tests passed, 0 tests failed out of 1 + +Total Test time (real) = 0.17 sec diff --git a/src/docs/sphinx/t_conduit_docs_tutorial_python_numeric_out.txt b/src/docs/sphinx/t_conduit_docs_tutorial_python_numeric_out.txt new file mode 100644 index 000000000..e2ead3087 --- /dev/null +++ b/src/docs/sphinx/t_conduit_docs_tutorial_python_numeric_out.txt @@ -0,0 +1,35 @@ +UpdateCTestConfiguration from :/Users/harrison37/Work/github/llnl/conduit/build-debug/DartConfiguration.tcl +Parse Config file:/Users/harrison37/Work/github/llnl/conduit/build-debug/DartConfiguration.tcl +UpdateCTestConfiguration from :/Users/harrison37/Work/github/llnl/conduit/build-debug/DartConfiguration.tcl +Parse Config file:/Users/harrison37/Work/github/llnl/conduit/build-debug/DartConfiguration.tcl +Test project /Users/harrison37/Work/github/llnl/conduit/build-debug +Constructing a list of tests +Done constructing a list of tests +Updating test list for fixtures +Added 0 tests to meet fixture requirements +Checking test dependency graph... +Checking test dependency graph end +test 107 + Start 107: t_conduit_docs_tutorial_python_numeric + +107: Test command: /Users/harrison37/Work/github/llnl/conduit/uberenv_libs/spack/opt/spack/darwin-mojave-x86_64/clang-10.0.0-apple/python-3.7.4-tuamu4qn6msg6bqa5hd7glbv5uxm5lup/bin/python3.7 "-B" "-m" "unittest" "-v" "t_conduit_docs_tutorial_python_numeric" +107: Environment variables: +107: PYTHONPATH=/Users/harrison37/Work/github/llnl/conduit/build-debug/python-modules/:/Users/harrison37/Work/github/llnl/conduit/src/tests/docs +107: Test timeout computed to be: 1500 +107: test_001_numeric_via_value (t_conduit_docs_tutorial_python_numeric.Conduit_Tutorial_Python_Numeric) ... ok +107: +107: ---------------------------------------------------------------------- +107: Ran 1 test in 0.000s +107: +107: OK +107: BEGIN_EXAMPLE("py_numeric_via_value") +107: 10 +107: END_EXAMPLE("py_numeric_via_value") +1/1 Test #107: t_conduit_docs_tutorial_python_numeric ... Passed 0.14 sec + +The following tests passed: + t_conduit_docs_tutorial_python_numeric + +100% tests passed, 0 tests failed out of 1 + +Total Test time (real) = 0.17 sec diff --git a/src/docs/sphinx/t_conduit_docs_tutorial_python_ownership_out.txt b/src/docs/sphinx/t_conduit_docs_tutorial_python_ownership_out.txt new file mode 100644 index 000000000..59dee2579 --- /dev/null +++ b/src/docs/sphinx/t_conduit_docs_tutorial_python_ownership_out.txt @@ -0,0 +1,57 @@ +UpdateCTestConfiguration from :/Users/harrison37/Work/github/llnl/conduit/build-debug/DartConfiguration.tcl +Parse Config file:/Users/harrison37/Work/github/llnl/conduit/build-debug/DartConfiguration.tcl +UpdateCTestConfiguration from :/Users/harrison37/Work/github/llnl/conduit/build-debug/DartConfiguration.tcl +Parse Config file:/Users/harrison37/Work/github/llnl/conduit/build-debug/DartConfiguration.tcl +Test project /Users/harrison37/Work/github/llnl/conduit/build-debug +Constructing a list of tests +Done constructing a list of tests +Updating test list for fixtures +Added 0 tests to meet fixture requirements +Checking test dependency graph... +Checking test dependency graph end +test 109 + Start 109: t_conduit_docs_tutorial_python_ownership + +109: Test command: /Users/harrison37/Work/github/llnl/conduit/uberenv_libs/spack/opt/spack/darwin-mojave-x86_64/clang-10.0.0-apple/python-3.7.4-tuamu4qn6msg6bqa5hd7glbv5uxm5lup/bin/python3.7 "-B" "-m" "unittest" "-v" "t_conduit_docs_tutorial_python_ownership" +109: Environment variables: +109: PYTHONPATH=/Users/harrison37/Work/github/llnl/conduit/build-debug/python-modules/:/Users/harrison37/Work/github/llnl/conduit/src/tests/docs +109: Test timeout computed to be: 1500 +109: test_001_mem_ownership_external (t_conduit_docs_tutorial_python_ownership.Conduit_Tutorial_Python_Ownership) ... ok +109: +109: ---------------------------------------------------------------------- +109: Ran 1 test in 0.001s +109: +109: OK +109: BEGIN_EXAMPLE("py_mem_ownership_external") +109: +109: mem_spaces: +109: 0x7fe138e8b760: +109: path: "v_owned" +109: type: "allocated" +109: bytes: 40 +109: 0x7fe138e8d3f0: +109: path: "v_external" +109: type: "external" +109: total_bytes_allocated: 40 +109: total_bytes_mmaped: 0 +109: total_bytes_compact: 80 +109: total_strided_bytes: 80 +109: +109: +109: v_owned: [0.0, 0.0, 0.0, 0.0, 0.0] +109: v_external: [0.0, 0.0, 0.0, 0.0, 0.0] +109: +109: +109: v_owned: [0.0, 0.0, 0.0, 0.0, 0.0] +109: v_external: [3.1415, 0.0, 0.0, 0.0, 0.0] +109: +109: [3.1415 0. 0. 0. 0. ] +109: END_EXAMPLE("py_mem_ownership_external") +1/1 Test #109: t_conduit_docs_tutorial_python_ownership ... Passed 0.15 sec + +The following tests passed: + t_conduit_docs_tutorial_python_ownership + +100% tests passed, 0 tests failed out of 1 + +Total Test time (real) = 0.17 sec diff --git a/src/docs/sphinx/t_conduit_docs_tutorial_python_parse_out.txt b/src/docs/sphinx/t_conduit_docs_tutorial_python_parse_out.txt new file mode 100644 index 000000000..e55cef54e --- /dev/null +++ b/src/docs/sphinx/t_conduit_docs_tutorial_python_parse_out.txt @@ -0,0 +1,73 @@ +UpdateCTestConfiguration from :/Users/harrison37/Work/github/llnl/conduit/build-debug/DartConfiguration.tcl +Parse Config file:/Users/harrison37/Work/github/llnl/conduit/build-debug/DartConfiguration.tcl +UpdateCTestConfiguration from :/Users/harrison37/Work/github/llnl/conduit/build-debug/DartConfiguration.tcl +Parse Config file:/Users/harrison37/Work/github/llnl/conduit/build-debug/DartConfiguration.tcl +Test project /Users/harrison37/Work/github/llnl/conduit/build-debug +Constructing a list of tests +Done constructing a list of tests +Updating test list for fixtures +Added 0 tests to meet fixture requirements +Checking test dependency graph... +Checking test dependency graph end +test 108 + Start 108: t_conduit_docs_tutorial_python_parse + +108: Test command: /Users/harrison37/Work/github/llnl/conduit/uberenv_libs/spack/opt/spack/darwin-mojave-x86_64/clang-10.0.0-apple/python-3.7.4-tuamu4qn6msg6bqa5hd7glbv5uxm5lup/bin/python3.7 "-B" "-m" "unittest" "-v" "t_conduit_docs_tutorial_python_parse" +108: Environment variables: +108: PYTHONPATH=/Users/harrison37/Work/github/llnl/conduit/build-debug/python-modules/:/Users/harrison37/Work/github/llnl/conduit/src/tests/docs +108: Test timeout computed to be: 1500 +108: test_docs_tutorial_json (t_conduit_docs_tutorial_python_parse.Conduit_Tutorial_Python_Parse) ... ok +108: test_docs_tutorial_yaml (t_conduit_docs_tutorial_python_parse.Conduit_Tutorial_Python_Parse) ... ok +108: test_docs_tutorial_yaml_inline_array (t_conduit_docs_tutorial_python_parse.Conduit_Tutorial_Python_Parse) ... ok +108: test_json_generator_pure_json (t_conduit_docs_tutorial_python_parse.Conduit_Tutorial_Python_Parse) ... ok +108: test_json_generator_pure_yaml (t_conduit_docs_tutorial_python_parse.Conduit_Tutorial_Python_Parse) ... ok +108: test_json_generator_std (t_conduit_docs_tutorial_python_parse.Conduit_Tutorial_Python_Parse) ... ok +108: +108: ---------------------------------------------------------------------- +108: Ran 6 tests in 0.001s +108: +108: OK +108: BEGIN_EXAMPLE("py_conduit_docs_tutorial_json") +108: 42.0 +108: +108: { +108: "mykey": {"dtype":"float64", "number_of_elements": 1, "offset": 0, "stride": 8, "element_bytes": 8, "endianness": "little"} +108: } +108: END_EXAMPLE("py_conduit_docs_tutorial_json") +108: BEGIN_EXAMPLE("py_conduit_docs_tutorial_yaml") +108: 42.0 +108: +108: { +108: "mykey": {"dtype":"float64", "number_of_elements": 1, "offset": 0, "stride": 8, "element_bytes": 8, "endianness": "little"} +108: } +108: END_EXAMPLE("py_conduit_docs_tutorial_yaml") +108: BEGIN_EXAMPLE("py_conduit_docs_tutorial_yaml_inline_array") +108: [ 0. 10. 20. 30.] +108: {"dtype":"float64", "number_of_elements": 4, "offset": 0, "stride": 8, "element_bytes": 8, "endianness": "little"} +108: END_EXAMPLE("py_conduit_docs_tutorial_yaml_inline_array") +108: BEGIN_EXAMPLE("py_json_generator_pure_json") +108: 100.0 +108: +108: test: 100.0 +108: +108: END_EXAMPLE("py_json_generator_pure_json") +108: BEGIN_EXAMPLE("py_json_generator_pure_yaml") +108: 100.0 +108: +108: test: 100.0 +108: +108: END_EXAMPLE("py_json_generator_pure_yaml") +108: BEGIN_EXAMPLE("py_json_generator_std") +108: 100.0 +108: +108: test: 100.0 +108: +108: END_EXAMPLE("py_json_generator_std") +1/1 Test #108: t_conduit_docs_tutorial_python_parse ... Passed 0.14 sec + +The following tests passed: + t_conduit_docs_tutorial_python_parse + +100% tests passed, 0 tests failed out of 1 + +Total Test time (real) = 0.17 sec diff --git a/src/docs/sphinx/t_conduit_docs_tutorial_python_relay_io_handle_examples_out.txt b/src/docs/sphinx/t_conduit_docs_tutorial_python_relay_io_handle_examples_out.txt new file mode 100644 index 000000000..e9ec1b3f0 --- /dev/null +++ b/src/docs/sphinx/t_conduit_docs_tutorial_python_relay_io_handle_examples_out.txt @@ -0,0 +1,64 @@ +UpdateCTestConfiguration from :/Users/harrison37/Work/github/llnl/conduit/build-debug/DartConfiguration.tcl +Parse Config file:/Users/harrison37/Work/github/llnl/conduit/build-debug/DartConfiguration.tcl +UpdateCTestConfiguration from :/Users/harrison37/Work/github/llnl/conduit/build-debug/DartConfiguration.tcl +Parse Config file:/Users/harrison37/Work/github/llnl/conduit/build-debug/DartConfiguration.tcl +Test project /Users/harrison37/Work/github/llnl/conduit/build-debug +Constructing a list of tests +Done constructing a list of tests +Updating test list for fixtures +Added 0 tests to meet fixture requirements +Checking test dependency graph... +Checking test dependency graph end +test 110 + Start 110: t_conduit_docs_tutorial_python_relay_io_handle_examples + +110: Test command: /Users/harrison37/Work/github/llnl/conduit/uberenv_libs/spack/opt/spack/darwin-mojave-x86_64/clang-10.0.0-apple/python-3.7.4-tuamu4qn6msg6bqa5hd7glbv5uxm5lup/bin/python3.7 "-B" "-m" "unittest" "-v" "t_conduit_docs_tutorial_python_relay_io_handle_examples" +110: Environment variables: +110: PYTHONPATH=/Users/harrison37/Work/github/llnl/conduit/build-debug/python-modules/:/Users/harrison37/Work/github/llnl/conduit/src/tests/docs +110: Test timeout computed to be: 1500 +110: test_001_io_handle (t_conduit_docs_tutorial_python_relay_io_handle_examples.Conduit_Tutorial_Python_Relay_IO_Handle) ... ok +110: +110: ---------------------------------------------------------------------- +110: Ran 1 test in 0.006s +110: +110: OK +110: +110: BEGIN_EXAMPLE("py_relay_io_handle") +110: +110: Node to write: +110: +110: a: +110: data: 1.0 +110: more_data: 2.0 +110: b: +110: my_string: "value" +110: +110: +110: Value at "a/data" = 1.0 +110: +110: Removed "a/more_data" +110: +110: Path "a/more_data" is no more +110: +110: Writing to "a/c" +110: +110: Children of "a": ['data', 'b', 'c'] +110: +110: Read Result: +110: +110: a: +110: data: 1.0 +110: b: +110: my_string: "value" +110: c: 42.0 +110: +110: +110: END_EXAMPLE("py_relay_io_handle") +1/1 Test #110: t_conduit_docs_tutorial_python_relay_io_handle_examples ... Passed 0.21 sec + +The following tests passed: + t_conduit_docs_tutorial_python_relay_io_handle_examples + +100% tests passed, 0 tests failed out of 1 + +Total Test time (real) = 0.23 sec diff --git a/src/docs/sphinx/t_conduit_docs_python_tutorial_relay_mpi_examples_out.txt b/src/docs/sphinx/t_conduit_docs_tutorial_python_relay_mpi_examples_out.txt similarity index 96% rename from src/docs/sphinx/t_conduit_docs_python_tutorial_relay_mpi_examples_out.txt rename to src/docs/sphinx/t_conduit_docs_tutorial_python_relay_mpi_examples_out.txt index dcc34eed2..a1ab52c5c 100644 --- a/src/docs/sphinx/t_conduit_docs_python_tutorial_relay_mpi_examples_out.txt +++ b/src/docs/sphinx/t_conduit_docs_tutorial_python_relay_mpi_examples_out.txt @@ -26,7 +26,6 @@ test 111 111: b: 111: my_string: "value" 111: -111: PyRelay_MPI_recv_using_schema 111: [rank: 1] received: 111: a: 111: data: 1.0 @@ -41,10 +40,8 @@ test 111 111: test_002_mpi_send_and_recv (t_conduit_docs_tutorial_python_relay_mpi_examples.Conduit_Tutorial_Python_Relay_IO_Handle) ... 111: BEGIN_EXAMPLE("py_mpi_send_and_recv") 111: [rank: 0] sending: [0, 1, 4, 9] -111: 111: [rank: 1] received: [0, 1, 4, 9] 111: -111: 111: END_EXAMPLE("py_mpi_send_and_recv") 111: ok 111: test_003_mpi_bcast_using_schema (t_conduit_docs_tutorial_python_relay_mpi_examples.Conduit_Tutorial_Python_Relay_IO_Handle) ... ok @@ -71,10 +68,8 @@ test 111 111: test_004_mpi_bcast_using_schema (t_conduit_docs_tutorial_python_relay_mpi_examples.Conduit_Tutorial_Python_Relay_IO_Handle) ... 111: BEGIN_EXAMPLE("py_mpi_bcast") 111: [rank: 0] broadcasting: [0, 1, 4, 9] -111: 111: [rank: 1] received: [0, 1, 4, 9] 111: -111: 111: END_EXAMPLE("py_mpi_bcast") 111: ok 111: test_005_mpi_sum_all_reduce (t_conduit_docs_tutorial_python_relay_mpi_examples.Conduit_Tutorial_Python_Relay_IO_Handle) ... ok @@ -82,7 +77,6 @@ test 111 111: BEGIN_EXAMPLE("py_mpi_sum_all_reduce") 111: [rank: 0] sum reduce result: [2, 2, 2, 2] 111: -111: 111: END_EXAMPLE("py_mpi_sum_all_reduce") 111: ok 111: test_006_mpi_all_gather (t_conduit_docs_tutorial_python_relay_mpi_examples.Conduit_Tutorial_Python_Relay_IO_Handle) ... ok @@ -97,20 +91,20 @@ test 111 111: ok 111: 111: ---------------------------------------------------------------------- -111: Ran 6 tests in 0.022s +111: Ran 6 tests in 0.083s 111: 111: OK 111: ok 111: 111: ---------------------------------------------------------------------- -111: Ran 6 tests in 0.022s +111: Ran 6 tests in 0.084s 111: 111: OK -1/1 Test #111: t_conduit_docs_tutorial_python_relay_mpi_examples ... Passed 0.28 sec +1/1 Test #111: t_conduit_docs_tutorial_python_relay_mpi_examples ... Passed 0.26 sec The following tests passed: t_conduit_docs_tutorial_python_relay_mpi_examples 100% tests passed, 0 tests failed out of 1 -Total Test time (real) = 0.31 sec +Total Test time (real) = 0.28 sec diff --git a/src/docs/sphinx/tutorial_cpp_basics.rst b/src/docs/sphinx/tutorial_cpp_basics.rst index 1852516f3..7135e2d97 100644 --- a/src/docs/sphinx/tutorial_cpp_basics.rst +++ b/src/docs/sphinx/tutorial_cpp_basics.rst @@ -56,25 +56,29 @@ Think of it as a hierarchical variant object. .. # from t_conduit_docs_tutorial_basics: basics_very_basic .. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_basics.cpp - :lines: 65-68 + :start-after: BEGIN_EXAMPLE("basics_very_basic") + :end-before: END_EXAMPLE("basics_very_basic") :language: cpp :dedent: 4 .. literalinclude:: t_conduit_docs_tutorial_basics_out.txt - :lines: 9-11 + :start-after: BEGIN_EXAMPLE("basics_very_basic") + :end-before: END_EXAMPLE("basics_very_basic") .. # from t_conduit_docs_tutorial_basics: basics_hierarchial The *Node* class supports hierarchical construction. .. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_basics.cpp - :lines: 76-84 + :start-after: BEGIN_EXAMPLE("basics_hierarchial") + :end-before: END_EXAMPLE("basics_hierarchial") :language: cpp :dedent: 4 .. literalinclude:: t_conduit_docs_tutorial_basics_out.txt - :lines: 19-30 + :start-after: BEGIN_EXAMPLE("basics_hierarchial") + :end-before: END_EXAMPLE("basics_hierarchial") Borrowing form JSON (and other similar notations), collections of named nodes are called *Objects* and collections of unnamed nodes are called *Lists*, all other types @@ -83,12 +87,14 @@ are leaves that represent concrete data. .. # from t_conduit_docs_tutorial_basics: basics_object_and_list .. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_basics.cpp - :lines: 92-105 + :start-after: BEGIN_EXAMPLE("basics_object_and_list") + :end-before: END_EXAMPLE("basics_object_and_list") :language: cpp :dedent: 4 .. literalinclude:: t_conduit_docs_tutorial_basics_out.txt - :lines: 38-53 + :start-after: BEGIN_EXAMPLE("basics_object_and_list") + :end-before: END_EXAMPLE("basics_object_and_list") You can use a *NodeIterator* ( or a *NodeConstIterator*) to iterate through a Node's children. @@ -96,12 +102,14 @@ children. .. # from t_conduit_docs_tutorial_basics: basics_object_and_list_itr .. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_basics.cpp - :lines: 113-142 + :start-after: BEGIN_EXAMPLE("basics_object_and_list_itr") + :end-before: END_EXAMPLE("basics_object_and_list_itr") :language: cpp :dedent: 4 .. literalinclude:: t_conduit_docs_tutorial_basics_out.txt - :lines: 61-84 + :start-after: BEGIN_EXAMPLE("basics_object_and_list_itr") + :end-before: END_EXAMPLE("basics_object_and_list_itr") Behind the scenes, *Node* instances manage a collection of memory spaces. @@ -109,12 +117,14 @@ Behind the scenes, *Node* instances manage a collection of memory spaces. .. # from t_conduit_docs_tutorial_basics: basics_mem_spaces .. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_basics.cpp - :lines: 151-160 + :start-after: BEGIN_EXAMPLE("basics_mem_spaces") + :end-before: END_EXAMPLE("basics_mem_spaces") :language: cpp :dedent: 4 .. literalinclude:: t_conduit_docs_tutorial_basics_out.txt - :lines: 92-118 + :start-after: BEGIN_EXAMPLE("basics_mem_spaces") + :end-before: END_EXAMPLE("basics_mem_spaces") .. # we could add an example here @@ -135,25 +145,31 @@ Conduit uses well defined bitwidth style types (inspired by NumPy) for leaf valu .. # from t_conduit_docs_tutorial_basics: basics_bw_style .. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_basics.cpp - :lines: 168-174 + :start-after: BEGIN_EXAMPLE("basics_bw_style") + :end-before: END_EXAMPLE("basics_bw_style") :language: cpp :dedent: 4 .. literalinclude:: t_conduit_docs_tutorial_basics_out.txt - :lines: 126-132 + :start-after: BEGIN_EXAMPLE("basics_bw_style") + :end-before: END_EXAMPLE("basics_bw_style") + Standard C++ numeric types will be mapped by the compiler to bitwidth style types. .. # from t_conduit_docs_tutorial_basics: basics_bw_style_from_native .. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_basics.cpp - :lines: 182-187 + :start-after: BEGIN_EXAMPLE("basics_bw_style_from_native") + :end-before: END_EXAMPLE("basics_bw_style_from_native") :language: cpp :dedent: 4 .. literalinclude:: t_conduit_docs_tutorial_basics_out.txt - :lines: 140-142 + :start-after: BEGIN_EXAMPLE("basics_bw_style_from_native") + :end-before: END_EXAMPLE("basics_bw_style_from_native") + Supported Bitwidth Style Types: diff --git a/src/docs/sphinx/tutorial_cpp_errors.rst b/src/docs/sphinx/tutorial_cpp_errors.rst index 8bfc98e23..361cc32c5 100644 --- a/src/docs/sphinx/tutorial_cpp_errors.rst +++ b/src/docs/sphinx/tutorial_cpp_errors.rst @@ -92,16 +92,20 @@ Here is an example that re-wires all three error handlers to print to standard o .. # from t_conduit_docs_tutorial_errors: custom range .. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_errors.cpp - :lines: 65-90 + :start-after: _conduit_error_handlers_funcs_start + :end-before: _conduit_error_handlers_funcs_end :language: cpp .. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_errors.cpp - :lines: 103-130 + :start-after: BEGIN_EXAMPLE("error_handlers_rewire") + :end-before: END_EXAMPLE("error_handlers_rewire") :language: cpp + :dedent: 4 -.. literalinclude:: t_conduit_docs_tutorial_errors_out.txt - :lines: 16-18 +.. literalinclude:: t_conduit_docs_tutorial_errors_out.txt + :start-after: BEGIN_EXAMPLE("error_handlers_rewire") + :end-before: END_EXAMPLE("error_handlers_rewire") Using Restoring Default Handlers @@ -111,7 +115,8 @@ The default handlers are part of the conduit::utils interface, so you can restor .. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_errors.cpp - :lines: 132-135 + :start-after: BEGIN_EXAMPLE("error_handlers_reset") + :end-before: END_EXAMPLE("error_handlers_reset") :language: cpp :dedent: 4 diff --git a/src/docs/sphinx/tutorial_cpp_json.rst b/src/docs/sphinx/tutorial_cpp_json.rst deleted file mode 100644 index 6a652e0a8..000000000 --- a/src/docs/sphinx/tutorial_cpp_json.rst +++ /dev/null @@ -1,192 +0,0 @@ -.. ############################################################################ -.. # Copyright (c) 2014-2019, Lawrence Livermore National Security, LLC. -.. # -.. # Produced at the Lawrence Livermore National Laboratory -.. # -.. # LLNL-CODE-666778 -.. # -.. # All rights reserved. -.. # -.. # This file is part of Conduit. -.. # -.. # For details, see: http://software.llnl.gov/conduit/. -.. # -.. # Please also read conduit/LICENSE -.. # -.. # Redistribution and use in source and binary forms, with or without -.. # modification, are permitted provided that the following conditions are met: -.. # -.. # * Redistributions of source code must retain the above copyright notice, -.. # this list of conditions and the disclaimer below. -.. # -.. # * Redistributions in binary form must reproduce the above copyright notice, -.. # this list of conditions and the disclaimer (as noted below) in the -.. # documentation and/or other materials provided with the distribution. -.. # -.. # * Neither the name of the LLNS/LLNL nor the names of its contributors may -.. # be used to endorse or promote products derived from this software without -.. # specific prior written permission. -.. # -.. # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -.. # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -.. # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -.. # ARE DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL SECURITY, -.. # LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE LIABLE FOR ANY -.. # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -.. # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -.. # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -.. # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -.. # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING -.. # IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -.. # POSSIBILITY OF SUCH DAMAGE. -.. # -.. ############################################################################ - -================================== -Reading YAML and JSON Strings -================================== - -Parsing text with *Node::parse()* ------------------------------------------------------- - -*Node::parse()* parses YAML and JSON strings into a *Node* tree. - -.. # from t_conduit_docs_tutorial_parse: t_conduit_docs_tutorial_yaml - -.. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_parse.cpp - :start-after: BEGIN_EXAMPLE("t_conduit_docs_tutorial_yaml") - :end-before: END_EXAMPLE("t_conduit_docs_tutorial_yaml") - :language: cpp - :dedent: 4 - -.. literalinclude:: t_conduit_docs_tutorial_parse_out.txt - :start-after: BEGIN_EXAMPLE("t_conduit_docs_tutorial_yaml") - :end-before: END_EXAMPLE("t_conduit_docs_tutorial_yaml") - -.. # from t_conduit_docs_tutorial_parse: t_conduit_docs_tutorial_json - -.. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_parse.cpp - :start-after: BEGIN_EXAMPLE("t_conduit_docs_tutorial_json") - :end-before: END_EXAMPLE("t_conduit_docs_tutorial_json") - :language: cpp - :dedent: 4 - -.. literalinclude:: t_conduit_docs_tutorial_parse_out.txt - :start-after: BEGIN_EXAMPLE("t_conduit_docs_tutorial_json") - :end-before: END_EXAMPLE("t_conduit_docs_tutorial_json") - -The first argument is the string to parse and the second argument -selects the protocol to use when parsing. - -Valid Protocols: ``json``, ``conduit_json``, ``conduit_base64_json``, ``yaml``. - -* ``json`` and ``yaml`` protocols parse pure JSON or YAML strings. For leaf - nodes wide types such as *int64*, *uint64*, and *float64* are inferred. - - -Homogeneous numeric lists are parsed as Conduit arrays. - -.. # from t_conduit_docs_tutorial_parse: t_conduit_docs_tutorial_yaml_inline_array - -.. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_parse.cpp - :start-after: BEGIN_EXAMPLE("t_conduit_docs_tutorial_yaml_inline_array") - :end-before: END_EXAMPLE("t_conduit_docs_tutorial_yaml_inline_array") - :language: cpp - :dedent: 4 - -.. literalinclude:: t_conduit_docs_tutorial_parse_out.txt - :start-after: BEGIN_EXAMPLE("t_conduit_docs_tutorial_yaml_inline_array") - :end-before: END_EXAMPLE("t_conduit_docs_tutorial_yaml_inline_array") - -* ``conduit_json`` parses JSON with conduit data type information, allowing you - to specify bitwidth style types, strides, etc. - -* ``conduit_base64_json`` combies the *conduit_json* protocol with an embedded - base64-encoded data block - -====================== -Generators -====================== - -Using *Generator* instances ---------------------------------------------------- -Node::parse() is sufficient for most use cases, but you can also use a *Generator* -instance to parse JSON and YAML. Additionally, Generators can parse a -conduit JSON schema and bind it to in-core data. - -.. # from t_conduit_docs_tutorial_json: json_generator_std - -.. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_parse.cpp - :start-after: BEGIN_EXAMPLE("t_json_generator_std") - :end-before: END_EXAMPLE("t_json_generator_std") - :language: cpp - :dedent: 4 - -.. literalinclude:: t_conduit_docs_tutorial_parse_out.txt - :start-after: BEGIN_EXAMPLE("t_json_generator_std") - :end-before: END_EXAMPLE("t_json_generator_std") - - -Like *Node::parse()*, *Generators* can also parse pure JSON or YAML. -For leaf nodes: wide types such as *int64*, *uint64*, and *float64* are inferred. - -.. # from t_conduit_docs_tutorial_json: json_generator_pure_json - -.. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_parse.cpp - :start-after: BEGIN_EXAMPLE("t_json_generator_pure_json") - :end-before: END_EXAMPLE("t_json_generator_pure_json") - :language: cpp - :dedent: 4 - -.. literalinclude:: t_conduit_docs_tutorial_parse_out.txt - :start-after: BEGIN_EXAMPLE("t_json_generator_pure_json") - :end-before: END_EXAMPLE("t_json_generator_pure_json") - -.. # from t_conduit_docs_tutorial_json: json_generator_pure_yaml - -.. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_parse.cpp - :start-after: BEGIN_EXAMPLE("t_json_generator_pure_yaml") - :end-before: END_EXAMPLE("t_json_generator_pure_yaml") - :language: cpp - :dedent: 4 - -.. literalinclude:: t_conduit_docs_tutorial_parse_out.txt - :start-after: BEGIN_EXAMPLE("t_json_generator_pure_yaml") - :end-before: END_EXAMPLE("t_json_generator_pure_yaml") - - - -Schemas can be bound to in-core data. - -.. # from t_conduit_docs_tutorial_json: json_generator_bind_to_incore - -.. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_parse.cpp - :start-after: BEGIN_EXAMPLE("t_json_generator_bind_to_incore") - :end-before: END_EXAMPLE("t_json_generator_bind_to_incore") - :language: cpp - :dedent: 4 - -.. literalinclude:: t_conduit_docs_tutorial_parse_out.txt - :start-after: BEGIN_EXAMPLE("t_json_generator_bind_to_incore") - :end-before: END_EXAMPLE("t_json_generator_bind_to_incore") - - - -Compacting Nodes --------------------------------- - -*Nodes* can be compacted to transform sparse data. - -.. # from t_conduit_docs_tutorial_json: json_generator_compact - -.. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_parse.cpp - :start-after: BEGIN_EXAMPLE("t_json_generator_compact") - :end-before: END_EXAMPLE("t_json_generator_compact") - :language: cpp - :dedent: 4 - -.. literalinclude:: t_conduit_docs_tutorial_parse_out.txt - :start-after: BEGIN_EXAMPLE("t_json_generator_compact") - :end-before: END_EXAMPLE("t_json_generator_compact") - - diff --git a/src/docs/sphinx/tutorial_cpp_numeric.rst b/src/docs/sphinx/tutorial_cpp_numeric.rst index 8cf061550..0049c5754 100644 --- a/src/docs/sphinx/tutorial_cpp_numeric.rst +++ b/src/docs/sphinx/tutorial_cpp_numeric.rst @@ -56,29 +56,29 @@ methods. .. # from t_conduit_docs_tutorial_numeric: numeric_as_dtype .. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_numeric.cpp - :start-after: _conduit_tutorial_cpp_numeric_node_as_dtype_start - :end-before: _conduit_tutorial_cpp_numeric_node_as_dtype_end + :start-after: BEGIN_EXAMPLE("numeric_as_dtype") + :end-before: END_EXAMPLE("numeric_as_dtype") :language: cpp :dedent: 4 .. literalinclude:: t_conduit_docs_tutorial_numeric_out.txt - :start-after: _conduit_tutorial_cpp_numeric_node_as_dtype_out_start - :end-before: _conduit_tutorial_cpp_numeric_node_as_dtype_out_end - + :start-after: BEGIN_EXAMPLE("numeric_as_dtype") + :end-before: END_EXAMPLE("numeric_as_dtype") + Or you can use Node::value(), which can infer the correct return type via a cast. .. # from t_conduit_docs_tutorial_numeric: numeric_via_value .. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_numeric.cpp - :start-after: _conduit_tutorial_cpp_numeric_node_value_start - :end-before: _conduit_tutorial_cpp_numeric_node_value_end + :start-after: BEGIN_EXAMPLE("numeric_via_value") + :end-before: END_EXAMPLE("numeric_via_value") :language: cpp :dedent: 4 .. literalinclude:: t_conduit_docs_tutorial_numeric_out.txt - :start-after: _conduit_tutorial_cpp_numeric_node_value_out_start - :end-before: _conduit_tutorial_cpp_numeric_node_value_out_end + :start-after: BEGIN_EXAMPLE("numeric_via_value") + :end-before: END_EXAMPLE("numeric_via_value") Accessing array data via pointers works the same way, using Node's *as_{type}* @@ -87,14 +87,14 @@ methods. .. # from t_conduit_docs_tutorial_numeric: numeric_ptr_as_dtype .. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_numeric.cpp - :start-after: _conduit_tutorial_cpp_numeric_array_node_pointer_start - :end-before: _conduit_tutorial_cpp_numeric_array_node_pointer_end + :start-after: BEGIN_EXAMPLE("numeric_ptr_as_dtype") + :end-before: END_EXAMPLE("numeric_ptr_as_dtype") :language: cpp :dedent: 4 .. literalinclude:: t_conduit_docs_tutorial_numeric_out.txt - :start-after: _conduit_tutorial_cpp_numeric_array_node_pointer_out_start - :end-before: _conduit_tutorial_cpp_numeric_array_node_pointer_out_end + :start-after: BEGIN_EXAMPLE("numeric_ptr_as_dtype") + :end-before: END_EXAMPLE("numeric_ptr_as_dtype") Or using Node::value(): @@ -102,14 +102,14 @@ Or using Node::value(): .. # from t_conduit_docs_tutorial_numeric: numeric_ptr_via_value .. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_numeric.cpp - :start-after: _conduit_tutorial_cpp_numeric_array_node_value_start - :end-before: _conduit_tutorial_cpp_numeric_array_node_value_end + :start-after: BEGIN_EXAMPLE("numeric_ptr_via_value") + :end-before: END_EXAMPLE("numeric_ptr_via_value") :language: cpp :dedent: 4 .. literalinclude:: t_conduit_docs_tutorial_numeric_out.txt - :start-after: _conduit_tutorial_cpp_numeric_array_node_value_out_start - :end-before: _conduit_tutorial_cpp_numeric_array_node_value_out_end + :start-after: BEGIN_EXAMPLE("numeric_ptr_via_value") + :end-before: END_EXAMPLE("numeric_ptr_via_value") @@ -120,15 +120,14 @@ indexing for all types of arrays. .. # from t_conduit_docs_tutorial_numeric: numeric_strided_data_array .. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_numeric.cpp - :start-after: _conduit_tutorial_cpp_numeric_non-contiguous_start - :end-before: _conduit_tutorial_cpp_numeric_non-contiguous_end + :start-after: BEGIN_EXAMPLE("numeric_strided_data_array") + :end-before: END_EXAMPLE("numeric_strided_data_array") :language: cpp :dedent: 4 .. literalinclude:: t_conduit_docs_tutorial_numeric_out.txt - :start-after: _conduit_tutorial_cpp_numeric_non-contiguous_out_start - :end-before: _conduit_tutorial_cpp_numeric_non-contiguous_out_end - + :start-after: BEGIN_EXAMPLE("numeric_strided_data_array") + :end-before: END_EXAMPLE("numeric_strided_data_array") Using Introspection and Conversion @@ -147,7 +146,8 @@ double array. :end-before: _conduit_tutorial_cpp_numeric_introspection_end :language: cpp + .. literalinclude:: t_conduit_docs_tutorial_numeric_out.txt - :start-after: _conduit_tutorial_cpp_numeric_introspection_out_start - :end-before: _conduit_tutorial_cpp_numeric_introspection_out_end + :start-after: BEGIN_EXAMPLE("numeric_double_conversion") + :end-before: END_EXAMPLE("numeric_double_conversion") diff --git a/src/docs/sphinx/tutorial_cpp_ownership.rst b/src/docs/sphinx/tutorial_cpp_ownership.rst index a1734f3eb..cbf6206a8 100644 --- a/src/docs/sphinx/tutorial_cpp_ownership.rst +++ b/src/docs/sphinx/tutorial_cpp_ownership.rst @@ -63,12 +63,14 @@ The **Node::set** methods support creating **owned** data and copying data value .. # from t_conduit_docs_tutorial_ownership: mem_ownership_external .. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_ownership.cpp - :lines: 64-82 + :start-after: BEGIN_EXAMPLE("mem_ownership_external") + :end-before: END_EXAMPLE("mem_ownership_external") :language: cpp :dedent: 4 .. literalinclude:: t_conduit_docs_tutorial_ownership_out.txt - :lines: 9-32 + :start-after: BEGIN_EXAMPLE("mem_ownership_external") + :end-before: END_EXAMPLE("mem_ownership_external") diff --git a/src/docs/sphinx/tutorial_python_basics.rst b/src/docs/sphinx/tutorial_python_basics.rst index c94dd55e7..f3c144598 100644 --- a/src/docs/sphinx/tutorial_python_basics.rst +++ b/src/docs/sphinx/tutorial_python_basics.rst @@ -56,12 +56,14 @@ Think of it as a hierarchical variant object. .. # from t_conduit_docs_tutorial_python_basics: basics_very_basic .. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_python_basics.py - :lines: 61-64 + :start-after: BEGIN_EXAMPLE("py_basics_very_basic") + :end-before: END_EXAMPLE("py_basics_very_basic") :language: python :dedent: 8 -.. literalinclude:: t_conduit_docs_python_tutorial_basics_out.txt - :lines: 28-30 +.. literalinclude:: t_conduit_docs_tutorial_python_basics_out.txt + :start-after: BEGIN_EXAMPLE("py_basics_very_basic") + :end-before: END_EXAMPLE("py_basics_very_basic") :dedent: 4 .. # from t_conduit_docs_tutorial_python_basics: basics_hierarchial @@ -69,13 +71,15 @@ Think of it as a hierarchical variant object. The *Node* class supports hierarchical construction. .. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_python_basics.py - :lines: 72-77 + :start-after: BEGIN_EXAMPLE("py_basics_hierarchial") + :end-before: END_EXAMPLE("py_basics_hierarchial") :language: python :dedent: 8 -.. literalinclude:: t_conduit_docs_python_tutorial_basics_out.txt - :lines: 36-47 +.. literalinclude:: t_conduit_docs_tutorial_python_basics_out.txt + :start-after: BEGIN_EXAMPLE("py_basics_hierarchial") + :end-before: END_EXAMPLE("py_basics_hierarchial") :dedent: 4 Borrowing form JSON (and other similar notations), collections of named nodes are @@ -85,12 +89,14 @@ are leaves that represent concrete data. .. # from t_conduit_docs_tutorial_python_basics: basics_object_and_list .. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_python_basics.py - :lines: 84-92 + :start-after: BEGIN_EXAMPLE("py_basics_object_and_list") + :end-before: END_EXAMPLE("py_basics_object_and_list") :language: python :dedent: 8 -.. literalinclude:: t_conduit_docs_python_tutorial_basics_out.txt - :lines: 54-69 +.. literalinclude:: t_conduit_docs_tutorial_python_basics_out.txt + :start-after: BEGIN_EXAMPLE("py_basics_object_and_list") + :end-before: END_EXAMPLE("py_basics_object_and_list") :dedent: 4 You can iterate through a Node's children. @@ -98,12 +104,14 @@ You can iterate through a Node's children. .. # from t_conduit_docs_tutorial_python_basics: basics_object_and_list_itr .. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_python_basics.py - :lines: 99-113 + :start-after: BEGIN_EXAMPLE("py_basics_object_and_list_itr") + :end-before: END_EXAMPLE("py_basics_object_and_list_itr") :language: python :dedent: 8 -.. literalinclude:: t_conduit_docs_python_tutorial_basics_out.txt - :lines: 75-98 +.. literalinclude:: t_conduit_docs_tutorial_python_basics_out.txt + :start-after: BEGIN_EXAMPLE("py_basics_object_and_list_itr") + :end-before: END_EXAMPLE("py_basics_object_and_list_itr") :dedent: 4 @@ -112,12 +120,14 @@ Behind the scenes, *Node* instances manage a collection of memory spaces. .. # from t_conduit_docs_tutorial_python_basics: basics_mem_spaces .. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_python_basics.py - :lines: 120-124 + :start-after: BEGIN_EXAMPLE("py_basics_mem_spaces") + :end-before: END_EXAMPLE("py_basics_mem_spaces") :language: python :dedent: 8 -.. literalinclude:: t_conduit_docs_python_tutorial_basics_out.txt - :lines: 104-130 +.. literalinclude:: t_conduit_docs_tutorial_python_basics_out.txt + :start-after: BEGIN_EXAMPLE("py_basics_mem_spaces") + :end-before: END_EXAMPLE("py_basics_mem_spaces") :dedent: 4 .. # we could add an example here @@ -139,12 +149,14 @@ are provided as NumPy ndarrays. .. # from t_conduit_docs_tutorial_python_basics: basics_bw_style .. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_python_basics.py - :lines: 131-133 + :start-after: BEGIN_EXAMPLE("py_basics_bw_style") + :end-before: END_EXAMPLE("py_basics_bw_style") :language: python :dedent: 8 -.. literalinclude:: t_conduit_docs_python_tutorial_basics_out.txt - :lines: 136-139 +.. literalinclude:: t_conduit_docs_tutorial_python_basics_out.txt + :start-after: BEGIN_EXAMPLE("py_basics_bw_style") + :end-before: END_EXAMPLE("py_basics_bw_style") :dedent: 4 Standard Python numeric types will be mapped to bitwidth style types. @@ -152,12 +164,14 @@ Standard Python numeric types will be mapped to bitwidth style types. .. # from t_conduit_docs_tutorial_python_basics: basics_bw_style_from_native .. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_python_basics.py - :lines: 141-143 + :start-after: BEGIN_EXAMPLE("py_basics_bw_style_from_native") + :end-before: END_EXAMPLE("py_basics_bw_style_from_native") :language: python :dedent: 8 -.. literalinclude:: t_conduit_docs_python_tutorial_basics_out.txt - :lines: 144-146 +.. literalinclude:: t_conduit_docs_tutorial_python_basics_out.txt + :start-after: BEGIN_EXAMPLE("py_basics_bw_style_from_native") + :end-before: END_EXAMPLE("py_basics_bw_style_from_native") :dedent: 4 Supported Bitwidth Style Types: diff --git a/src/docs/sphinx/tutorial_python_parse.rst b/src/docs/sphinx/tutorial_python_parse.rst index df6180729..d196fd572 100644 --- a/src/docs/sphinx/tutorial_python_parse.rst +++ b/src/docs/sphinx/tutorial_python_parse.rst @@ -52,25 +52,25 @@ Parsing text with *Node::parse()* *Node.parse()* parses YAML and JSON strings into a *Node* tree. .. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_python_parse.py - :start-after: BEGIN_EXAMPLE("t_py_conduit_docs_tutorial_yaml") - :end-before: END_EXAMPLE("t_py_conduit_docs_tutorial_yaml") + :start-after: BEGIN_EXAMPLE("py_conduit_docs_tutorial_yaml") + :end-before: END_EXAMPLE("py_conduit_docs_tutorial_yaml") :language: python :dedent: 8 -.. literalinclude:: t_conduit_docs_python_tutorial_parse_out.txt - :start-after: BEGIN_EXAMPLE("t_py_conduit_docs_tutorial_yaml") - :end-before: END_EXAMPLE("t_py_conduit_docs_tutorial_yaml") +.. literalinclude:: t_conduit_docs_tutorial_python_parse_out.txt + :start-after: BEGIN_EXAMPLE("py_conduit_docs_tutorial_yaml") + :end-before: END_EXAMPLE("py_conduit_docs_tutorial_yaml") :dedent: 4 .. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_python_parse.py - :start-after: BEGIN_EXAMPLE("t_py_conduit_docs_tutorial_json") - :end-before: END_EXAMPLE("t_py_conduit_docs_tutorial_json") + :start-after: BEGIN_EXAMPLE("py_conduit_docs_tutorial_json") + :end-before: END_EXAMPLE("py_conduit_docs_tutorial_json") :language: python :dedent: 8 -.. literalinclude:: t_conduit_docs_python_tutorial_parse_out.txt - :start-after: BEGIN_EXAMPLE("t_py_conduit_docs_tutorial_json") - :end-before: END_EXAMPLE("t_py_conduit_docs_tutorial_json") +.. literalinclude:: t_conduit_docs_tutorial_python_parse_out.txt + :start-after: BEGIN_EXAMPLE("py_conduit_docs_tutorial_json") + :end-before: END_EXAMPLE("py_conduit_docs_tutorial_json") :dedent: 4 The first argument is the string to parse and the second argument @@ -85,14 +85,14 @@ Valid Protocols: ``json``, ``conduit_json``, ``conduit_base64_json``, ``yaml` Homogeneous numeric lists are parsed as Conduit arrays. .. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_python_parse.py - :start-after: BEGIN_EXAMPLE("t_py_conduit_docs_tutorial_yaml_inline_array") - :end-before: END_EXAMPLE("t_py_conduit_docs_tutorial_yaml_inline_array") + :start-after: BEGIN_EXAMPLE("py_conduit_docs_tutorial_yaml_inline_array") + :end-before: END_EXAMPLE("py_conduit_docs_tutorial_yaml_inline_array") :language: python :dedent: 8 -.. literalinclude:: t_conduit_docs_python_tutorial_parse_out.txt - :start-after: BEGIN_EXAMPLE("t_py_conduit_docs_tutorial_yaml_inline_array") - :end-before: END_EXAMPLE("t_py_conduit_docs_tutorial_yaml_inline_array") +.. literalinclude:: t_conduit_docs_tutorial_python_parse_out.txt + :start-after: BEGIN_EXAMPLE("py_conduit_docs_tutorial_yaml_inline_array") + :end-before: END_EXAMPLE("py_conduit_docs_tutorial_yaml_inline_array") :dedent: 4 * ``conduit_json`` parses JSON with conduit data type information, allowing you @@ -112,14 +112,14 @@ instance to parse JSON and YAML. Additionally, Generators can parse a conduit JSON schema and bind it to in-core data. .. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_python_parse.py - :start-after: BEGIN_EXAMPLE("t_py_json_generator_std") - :end-before: END_EXAMPLE("t_py_json_generator_std") + :start-after: BEGIN_EXAMPLE("py_json_generator_std") + :end-before: END_EXAMPLE("py_json_generator_std") :language: python :dedent: 8 -.. literalinclude:: t_conduit_docs_python_tutorial_parse_out.txt - :start-after: BEGIN_EXAMPLE("t_py_json_generator_std") - :end-before: END_EXAMPLE("t_py_json_generator_std") +.. literalinclude:: t_conduit_docs_tutorial_python_parse_out.txt + :start-after: BEGIN_EXAMPLE("py_json_generator_std") + :end-before: END_EXAMPLE("py_json_generator_std") :dedent: 4 Like *Node::parse()*, *Generators* can also parse pure JSON or YAML. @@ -127,25 +127,25 @@ For leaf nodes: wide types such as *int64*, *uint64*, and *float64* are inferred .. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_python_parse.py - :start-after: BEGIN_EXAMPLE("t_py_json_generator_pure_json") - :end-before: END_EXAMPLE("t_py_json_generator_pure_json") + :start-after: BEGIN_EXAMPLE("py_json_generator_pure_json") + :end-before: END_EXAMPLE("py_json_generator_pure_json") :language: python :dedent: 8 -.. literalinclude:: t_conduit_docs_tutorial_parse_out.txt - :start-after: BEGIN_EXAMPLE("t_py_json_generator_pure_json") - :end-before: END_EXAMPLE("t_py_json_generator_pure_json") +.. literalinclude:: t_conduit_docs_tutorial_python_parse_out.txt + :start-after: BEGIN_EXAMPLE("py_json_generator_pure_json") + :end-before: END_EXAMPLE("py_json_generator_pure_json") :dedent: 4 .. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_python_parse.py - :start-after: BEGIN_EXAMPLE("t_py_json_generator_pure_yaml") - :end-before: END_EXAMPLE("t_py_json_generator_pure_yaml") + :start-after: BEGIN_EXAMPLE("py_json_generator_pure_yaml") + :end-before: END_EXAMPLE("py_json_generator_pure_yaml") :language: python :dedent: 8 -.. literalinclude:: t_conduit_docs_python_tutorial_parse_out.txt - :start-after: BEGIN_EXAMPLE("t_py_json_generator_pure_yaml") - :end-before: END_EXAMPLE("t_py_json_generator_pure_yaml") +.. literalinclude:: t_conduit_docs_tutorial_python_parse_out.txt + :start-after: BEGIN_EXAMPLE("py_json_generator_pure_yaml") + :end-before: END_EXAMPLE("py_json_generator_pure_yaml") :dedent: 4 .. ====================== @@ -160,14 +160,14 @@ For leaf nodes: wide types such as *int64*, *uint64*, and *float64* are inferred .. .. .. .. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_python_parse.py -.. :start-after: BEGIN_EXAMPLE("t_py_json_generator_std") -.. :end-before: END_EXAMPLE("t_py_json_generator_std") +.. :start-after: BEGIN_EXAMPLE("py_json_generator_std") +.. :end-before: END_EXAMPLE("py_json_generator_std") .. :language: python .. :dedent: 8 .. .. .. literalinclude:: t_conduit_docs_python_tutorial_parse_out.txt -.. :start-after: BEGIN_EXAMPLE("t_py_json_generator_std") -.. :end-before: END_EXAMPLE("t_py_json_generator_std") +.. :start-after: BEGIN_EXAMPLE("py_json_generator_std") +.. :end-before: END_EXAMPLE("py_json_generator_std") .. :dedent: 4 .. .. @@ -176,14 +176,14 @@ For leaf nodes: wide types such as *int64*, *uint64*, and *float64* are inferred .. .. # from t_conduit_docs_tutorial_python_json: json_generator_pure_json .. .. .. literalinclude:: ../../tests/docs/t_conduit_docs_tutorial_python_parse.py -.. :start-after: BEGIN_EXAMPLE("t_py_json_generator_pure_json") -.. :end-before: END_EXAMPLE("t_py_json_generator_pure_json") +.. :start-after: BEGIN_EXAMPLE("py_json_generator_pure_json") +.. :end-before: END_EXAMPLE("py_json_generator_pure_json") .. :language: python .. :dedent: 8 .. .. .. literalinclude:: t_conduit_docs_python_tutorial_parse_out.txt -.. :start-after: BEGIN_EXAMPLE("t_py_json_generator_pure_json") -.. :end-before: END_EXAMPLE("t_py_json_generator_pure_json") +.. :start-after: BEGIN_EXAMPLE("py_json_generator_pure_json") +.. :end-before: END_EXAMPLE("py_json_generator_pure_json") .. :dedent: 4 diff --git a/src/libs/conduit/conduit_data_array.cpp b/src/libs/conduit/conduit_data_array.cpp index 9a2860e69..4868739a6 100644 --- a/src/libs/conduit/conduit_data_array.cpp +++ b/src/libs/conduit/conduit_data_array.cpp @@ -319,13 +319,49 @@ DataArray::diff_compatible(const DataArray &array, Node &info, const float return res; } +//---------------------------------------------------------------------------// +template +std::string +DataArray::to_string(const std::string &protocol) const +{ + std::ostringstream oss; + to_string_stream(oss,protocol); + return oss.str(); +} + +//---------------------------------------------------------------------------// +template +void +DataArray::to_string_stream(std::ostream &os, + const std::string &protocol) const +{ + if(protocol != "json") + { + // unsupported + CONDUIT_ERROR("Unknown DataType::to_string protocol:" << protocol + <<"\nSupported protocols:\n" + <<" json"); + } + + to_json_stream(os); +} + +//---------------------------------------------------------------------------// +template +std::string +DataArray::to_string_default() const +{ + return to_string(); +} + + //---------------------------------------------------------------------------// template std::string DataArray::to_json() const { std::ostringstream oss; - to_json(oss); + to_json_stream(oss); return oss.str(); } @@ -333,6 +369,13 @@ DataArray::to_json() const template void DataArray::to_json(std::ostream &os) const +{ + to_json_stream(os); +} +//---------------------------------------------------------------------------// +template +void +DataArray::to_json_stream(std::ostream &os) const { index_t nele = number_of_elements(); if(nele > 1) diff --git a/src/libs/conduit/conduit_data_array.hpp b/src/libs/conduit/conduit_data_array.hpp index e79b44fb9..fe0b5d6d6 100644 --- a/src/libs/conduit/conduit_data_array.hpp +++ b/src/libs/conduit/conduit_data_array.hpp @@ -203,8 +203,22 @@ class CONDUIT_API DataArray //----------------------------------------------------------------------------- // Transforms //----------------------------------------------------------------------------- + std::string to_string(const std::string &protocol="json") const; + void to_string_stream(std::ostream &os, + const std::string &protocol="json") const; + + // NOTE(cyrush): The primary reason this function exists is to enable + // easier compatibility with debugging tools (e.g. totalview, gdb) that + // have difficulty allocating default string parameters. + std::string to_string_default() const; + std::string to_json() const; + + /// DEPRECATED: to_json(std::ostream &os) is deprecated in favor of + /// to_json_stream(std::ostream &os) void to_json(std::ostream &os) const; + + void to_json_stream(std::ostream &os) const; void compact_elements_to(uint8 *data) const; //----------------------------------------------------------------------------- diff --git a/src/libs/conduit/conduit_data_type.cpp b/src/libs/conduit/conduit_data_type.cpp index 9cd036547..2db9a14d2 100644 --- a/src/libs/conduit/conduit_data_type.cpp +++ b/src/libs/conduit/conduit_data_type.cpp @@ -1243,6 +1243,38 @@ DataType::default_bytes(const std::string &name) // Transforms //----------------------------------------------------------------------------- +//---------------------------------------------------------------------------// +std::string +DataType::to_string(const std::string &protocol) const +{ + std::ostringstream oss; + to_string_stream(oss,protocol); + return oss.str(); +} + +//---------------------------------------------------------------------------// +void +DataType::to_string_stream(std::ostream &os, + const std::string &protocol) const +{ + if(protocol != "json") + { + // unsupported + CONDUIT_ERROR("Unknown DataType::to_string protocol:" << protocol + <<"\nSupported protocols:\n" + <<" json"); + } + + to_json_stream(os); +} + +//---------------------------------------------------------------------------// +std::string +DataType::to_string_default() const +{ + return to_string(); +} + //---------------------------------------------------------------------------// std::string DataType::to_json() const diff --git a/src/libs/conduit/conduit_data_type.hpp b/src/libs/conduit/conduit_data_type.hpp index 3a2692398..c93984636 100644 --- a/src/libs/conduit/conduit_data_type.hpp +++ b/src/libs/conduit/conduit_data_type.hpp @@ -525,7 +525,16 @@ class CONDUIT_API DataType //----------------------------------------------------------------------------- // Transforms //----------------------------------------------------------------------------- - std::string to_json() const; + std::string to_string(const std::string &protocol="json") const; + void to_string_stream(std::ostream &os, + const std::string &protocol="json") const; + + // NOTE(cyrush): The primary reason this function exists is to enable + // easier compatibility with debugging tools (e.g. totalview, gdb) that + // have difficulty allocating default string parameters. + std::string to_string_default() const; + + std::string to_json() const; void to_json_stream(std::ostream &os) const; void compact_to(DataType &dtype) const; diff --git a/src/libs/conduit/conduit_node.cpp b/src/libs/conduit/conduit_node.cpp index 0bd455a2a..a51bfb372 100644 --- a/src/libs/conduit/conduit_node.cpp +++ b/src/libs/conduit/conduit_node.cpp @@ -11217,6 +11217,66 @@ Node::ConstValue::operator const long_double_array() const // End Node::ConstValue //----------------------------------------------------------------------------- +//----------------------------------------------------------------------------- +// -- String construction methods --- +//----------------------------------------------------------------------------- + +//----------------------------------------------------------------------------- +std::string +Node::to_string(const std::string &protocol, + index_t indent, + index_t depth, + const std::string &pad, + const std::string &eoe) const +{ + std::ostringstream oss; + to_string_stream(oss,protocol,indent,depth,pad,eoe); + return oss.str(); +} + +//----------------------------------------------------------------------------- +void +Node::to_string_stream(std::ostream &os, + const std::string &protocol, + index_t indent, + index_t depth, + const std::string &pad, + const std::string &eoe) const +{ + if(protocol == "yaml") + { + to_yaml_stream(os,protocol,indent,depth,pad,eoe); + } + else // assume json + { + to_json_stream(os,protocol,indent,depth,pad,eoe); + } +} + +//----------------------------------------------------------------------------- +void +Node::to_string_stream(const std::string &stream_path, + const std::string &protocol, + index_t indent, + index_t depth, + const std::string &pad, + const std::string &eoe) const +{ + std::ofstream ofs; + ofs.open(stream_path.c_str()); + if(!ofs.is_open()) + CONDUIT_ERROR("Node::to_string_stream failed to open: " << stream_path); + to_string_stream(ofs,protocol,indent,depth,pad,eoe); + ofs.close(); +} + +//----------------------------------------------------------------------------- +std::string +Node::to_string_default() const +{ + return to_string(); +} + //----------------------------------------------------------------------------- // -- JSON construction methods --- //----------------------------------------------------------------------------- @@ -11243,7 +11303,11 @@ Node::to_json(const std::string &protocol, } else { - CONDUIT_ERROR("Unknown to_json protocol:" << protocol); + CONDUIT_ERROR("Unknown Node::to_json protocol:" << protocol + << "\nSupported protocols:\n" + << " json\n" + << " conduit_json\n" + << " conduit_base64_json\n"); } return "{}"; @@ -11272,7 +11336,11 @@ Node::to_json_stream(const std::string &stream_path, } else { - CONDUIT_ERROR("Unknown to_json protocol:" << protocol); + CONDUIT_ERROR("Unknown Node::to_json protocol:" << protocol + << "\nSupported protocols:\n" + << " json\n" + << " conduit_json\n" + << " conduit_base64_json\n"); } } @@ -11299,7 +11367,11 @@ Node::to_json_stream(std::ostream &os, } else { - CONDUIT_ERROR("Unknown to_json protocol:" << protocol); + CONDUIT_ERROR("Unknown Node::to_json protocol:" << protocol + << "\nSupported protocols:\n" + << " json\n" + << " conduit_json\n" + << " conduit_base64_json\n"); } } @@ -11328,7 +11400,9 @@ Node::to_yaml(const std::string &protocol, } else { - CONDUIT_ERROR("Unknown to_yaml protocol:" << protocol); + CONDUIT_ERROR("Unknown Node::to_yaml protocol:" << protocol + << "\nSupported protocols:\n" + << " yaml\n"); } return "{}"; @@ -11349,9 +11423,12 @@ Node::to_yaml_stream(const std::string &stream_path, } else { - CONDUIT_ERROR("Unknown to_yaml protocol:" << protocol); + CONDUIT_ERROR("Unknown Node::to_yaml protocol:" << protocol + << "\nSupported protocols:\n" + << " yaml\n"); } } + //----------------------------------------------------------------------------- void Node::to_yaml_stream(std::ostream &os, @@ -11367,7 +11444,9 @@ Node::to_yaml_stream(std::ostream &os, } else { - CONDUIT_ERROR("Unknown to_yaml protocol:" << protocol); + CONDUIT_ERROR("Unknown Node::to_yaml protocol:" << protocol + << "\nSupported protocols:\n" + << " yaml\n"); } } @@ -11408,7 +11487,7 @@ Node::to_json_generic(const std::string &stream_path, std::ofstream ofs; ofs.open(stream_path.c_str()); if(!ofs.is_open()) - CONDUIT_ERROR(" failed to open: " << stream_path); + CONDUIT_ERROR("Node::to_json failed to open: " << stream_path); to_json_generic(ofs,detailed,indent,depth,pad,eoe); ofs.close(); } @@ -11493,36 +11572,36 @@ Node::to_json_generic(std::ostream &os, { // ints case DataType::INT8_ID: - as_int8_array().to_json(os); + as_int8_array().to_json_stream(os); break; case DataType::INT16_ID: - as_int16_array().to_json(os); + as_int16_array().to_json_stream(os); break; case DataType::INT32_ID: - as_int32_array().to_json(os); + as_int32_array().to_json_stream(os); break; case DataType::INT64_ID: - as_int64_array().to_json(os); + as_int64_array().to_json_stream(os); break; // uints case DataType::UINT8_ID: - as_uint8_array().to_json(os); + as_uint8_array().to_json_stream(os); break; case DataType::UINT16_ID: - as_uint16_array().to_json(os); + as_uint16_array().to_json_stream(os); break; case DataType::UINT32_ID: - as_uint32_array().to_json(os); + as_uint32_array().to_json_stream(os); break; case DataType::UINT64_ID: - as_uint64_array().to_json(os); + as_uint64_array().to_json_stream(os); break; // floats case DataType::FLOAT32_ID: - as_float32_array().to_json(os); + as_float32_array().to_json_stream(os); break; case DataType::FLOAT64_ID: - as_float64_array().to_json(os); + as_float64_array().to_json_stream(os); break; // char8_str case DataType::CHAR8_STR_ID: @@ -11687,7 +11766,7 @@ Node::to_base64_json(std::ostream &os, utils::indent(os,indent,depth+1,pad); os << "\"schema\": "; - n.schema().to_json_stream(os,true,indent,depth+1,pad,eoe); + n.schema().to_json_stream(os,indent,depth+1,pad,eoe); os << "," << eoe; @@ -11769,6 +11848,10 @@ Node::to_yaml_generic(std::ostream &os, depth+1, pad, eoe); + + // if the child is a leaf, we need eoe + if(m_children[i]->number_of_children() == 0) + os << eoe; } } else if(dtype().id() == DataType::LIST_ID) @@ -11785,6 +11868,10 @@ Node::to_yaml_generic(std::ostream &os, depth+1, pad, eoe); + + // if the child is a leaf, we need eoe + if(m_children[i]->number_of_children() == 0) + os << eoe; } } else // assume leaf data type @@ -11793,36 +11880,36 @@ Node::to_yaml_generic(std::ostream &os, { // ints case DataType::INT8_ID: - as_int8_array().to_json(os); + as_int8_array().to_json_stream(os); break; case DataType::INT16_ID: - as_int16_array().to_json(os); + as_int16_array().to_json_stream(os); break; case DataType::INT32_ID: - as_int32_array().to_json(os); + as_int32_array().to_json_stream(os); break; case DataType::INT64_ID: - as_int64_array().to_json(os); + as_int64_array().to_json_stream(os); break; // uints case DataType::UINT8_ID: - as_uint8_array().to_json(os); + as_uint8_array().to_json_stream(os); break; case DataType::UINT16_ID: - as_uint16_array().to_json(os); + as_uint16_array().to_json_stream(os); break; case DataType::UINT32_ID: - as_uint32_array().to_json(os); + as_uint32_array().to_json_stream(os); break; case DataType::UINT64_ID: - as_uint64_array().to_json(os); + as_uint64_array().to_json_stream(os); break; // floats case DataType::FLOAT32_ID: - as_float32_array().to_json(os); + as_float32_array().to_json_stream(os); break; case DataType::FLOAT64_ID: - as_float64_array().to_json(os); + as_float64_array().to_json_stream(os); break; // char8_str case DataType::CHAR8_STR_ID: @@ -11833,10 +11920,7 @@ Node::to_yaml_generic(std::ostream &os, // empty case DataType::EMPTY_ID: break; - } - - os << eoe; } os.flags(prev_stream_flags); @@ -11927,7 +12011,7 @@ Node::info()const void Node::print() const { - to_json_stream(std::cout); + to_string_stream(std::cout); std::cout << std::endl; } @@ -11935,7 +12019,7 @@ Node::print() const void Node::print_detailed() const { - to_json_stream(std::cout,"conduit_json"); + to_string_stream(std::cout,"conduit_json"); std::cout << std::endl; } diff --git a/src/libs/conduit/conduit_node.hpp b/src/libs/conduit/conduit_node.hpp index 6742c4bb6..fa6eed623 100644 --- a/src/libs/conduit/conduit_node.hpp +++ b/src/libs/conduit/conduit_node.hpp @@ -3323,6 +3323,38 @@ class CONDUIT_API Node { return ConstValue(this,true); } +//----------------------------------------------------------------------------- +// -- String construction methods --- +//----------------------------------------------------------------------------- + // accepted protocols: + // "json" + // "conduit_json" + // "conduit_base64_json" + // "yaml" + std::string to_string(const std::string &protocol="yaml", + index_t indent=2, + index_t depth=0, + const std::string &pad=" ", + const std::string &eoe="\n") const; + + void to_string_stream(std::ostream &os, + const std::string &protocol="yaml", + index_t indent=2, + index_t depth=0, + const std::string &pad=" ", + const std::string &eoe="\n") const; + + void to_string_stream(const std::string &stream_path, + const std::string &protocol="json", + index_t indent=2, + index_t depth=0, + const std::string &pad=" ", + const std::string &eoe="\n") const; + + // NOTE(cyrush): The primary reason this function exists is to enable easier + // compatibility with debugging tools (e.g. totalview, gdb) that have + // difficulty allocating default string parameters. + std::string to_string_default() const; //----------------------------------------------------------------------------- // -- JSON construction methods --- diff --git a/src/libs/conduit/conduit_schema.cpp b/src/libs/conduit/conduit_schema.cpp index a3fa8663a..d35827840 100644 --- a/src/libs/conduit/conduit_schema.cpp +++ b/src/libs/conduit/conduit_schema.cpp @@ -486,24 +486,120 @@ Schema::compact_to(Schema &s_dest) const compact_to(s_dest,0); } + //---------------------------------------------------------------------------// std::string -Schema::to_json(bool detailed, - index_t indent, - index_t depth, - const std::string &pad, - const std::string &eoe) const +Schema::to_string(const std::string &protocol, + index_t indent, + index_t depth, + const std::string &pad, + const std::string &eoe) const +{ + std::ostringstream oss; + to_string_stream(oss,protocol,indent,depth,pad,eoe); + return oss.str(); +} + +//---------------------------------------------------------------------------// +void +Schema::to_string_stream(std::ostream &os, + const std::string &protocol, + index_t indent, + index_t depth, + const std::string &pad, + const std::string &eoe) const +{ + if(protocol != "json") + { + // unsupported + CONDUIT_ERROR("Unknown Schema::to_string protocol:" << protocol + <<"\nSupported protocols:\n" + <<" json"); + } + + return to_json_stream(os,indent,depth,pad,eoe); +} + + +//---------------------------------------------------------------------------// +void +Schema::to_string_stream(const std::string &stream_path, + const std::string &protocol, + index_t indent, + index_t depth, + const std::string &pad, + const std::string &eoe) const +{ + std::ofstream ofs; + ofs.open(stream_path.c_str()); + if(!ofs.is_open()) + CONDUIT_ERROR("Schema::to_string_stream failed to open: " << stream_path); + to_string_stream(ofs,protocol,indent,depth,pad,eoe); + ofs.close(); +} + +//---------------------------------------------------------------------------// +std::string +Schema::to_string_default() const +{ + return to_string(); +} + +//---------------------------------------------------------------------------// +// note: variants with detailed arg (which hasn't been used) are deprecated +std::string +Schema::to_json(bool /* detailed */, // unused + index_t indent, + index_t depth, + const std::string &pad, + const std::string &eoe) const +{ + return to_json(indent,depth,pad,eoe); +} + +//---------------------------------------------------------------------------// +// note: variants with detailed arg (which hasn't been used) are deprecated +void +Schema::to_json_stream(std::ostream &os, + bool /* detailed */, // unused + index_t indent, + index_t depth, + const std::string &pad, + const std::string &eoe) const +{ + to_json_stream(os,indent,depth,pad,eoe); +} + +//---------------------------------------------------------------------------// +// note: variants with detailed arg (which hasn't been used) are deprecated +void +Schema::to_json_stream(const std::string &stream_path, + bool /* detailed */, // unused + index_t indent, + index_t depth, + const std::string &pad, + const std::string &eoe) const +{ + to_json_stream(stream_path,indent,depth,pad,eoe); +} + + +//---------------------------------------------------------------------------// +std::string +Schema::to_json(index_t indent, + index_t depth, + const std::string &pad, + const std::string &eoe) const { std::ostringstream oss; - to_json_stream(oss,detailed,indent,depth,pad,eoe); + to_json_stream(oss,indent,depth,pad,eoe); return oss.str(); } //---------------------------------------------------------------------------// void Schema::to_json_stream(std::ostream &os, - bool detailed, - index_t indent, + index_t indent, index_t depth, const std::string &pad, const std::string &eoe) const @@ -519,7 +615,7 @@ Schema::to_json_stream(std::ostream &os, { utils::indent(os,indent,depth+1,pad); os << "\""<< object_order()[i] << "\": "; - children()[i]->to_json_stream(os,detailed,indent,depth+1,pad,eoe); + children()[i]->to_json_stream(os,indent,depth+1,pad,eoe); if(i < nchildren-1) os << ","; os << eoe; @@ -537,13 +633,13 @@ Schema::to_json_stream(std::ostream &os, for(size_t i=0; i < nchildren;i++) { utils::indent(os,indent,depth+1,pad); - children()[i]->to_json_stream(os,detailed,indent,depth+1,pad,eoe); + children()[i]->to_json_stream(os,indent,depth+1,pad,eoe); if(i < nchildren-1) os << ","; os << eoe; } utils::indent(os,indent,depth,pad); - os << "]"; + os << "]"; } else // assume leaf data type { @@ -554,7 +650,6 @@ Schema::to_json_stream(std::ostream &os, //---------------------------------------------------------------------------// void Schema::to_json_stream(const std::string &stream_path, - bool detailed, index_t indent, index_t depth, const std::string &pad, @@ -563,8 +658,8 @@ Schema::to_json_stream(const std::string &stream_path, std::ofstream ofs; ofs.open(stream_path.c_str()); if(!ofs.is_open()) - CONDUIT_ERROR(" failed to open: " << stream_path); - to_json_stream(ofs,detailed,indent,depth,pad,eoe); + CONDUIT_ERROR("Schema::to_json_stream failed to open: " << stream_path); + to_json_stream(ofs,indent,depth,pad,eoe); ofs.close(); } @@ -582,17 +677,30 @@ Schema::to_json_default() const //----------------------------------------------------------------------------- //---------------------------------------------------------------------------// -void +// +// note: variants with detailed arg (which hasn't been used) are deprecated +void +Schema::save(const std::string &ofname, + bool /* detailed*/, // unused + index_t indent, + index_t depth, + const std::string &pad, + const std::string &eoe) const +{ + save(ofname,indent,depth,pad,eoe); +} + +//---------------------------------------------------------------------------// +void Schema::save(const std::string &ofname, - bool detailed, - index_t indent, + index_t indent, index_t depth, const std::string &pad, const std::string &eoe) const { // TODO: this is ineff, get base class rep correct? std::ostringstream oss; - to_json_stream(oss,detailed,indent,depth,pad,eoe); + to_json_stream(oss,indent,depth,pad,eoe); std::ofstream ofile; ofile.open(ofname.c_str()); @@ -602,6 +710,7 @@ Schema::save(const std::string &ofname, ofile.close(); } + //---------------------------------------------------------------------------// void Schema::load(const std::string &ifname) diff --git a/src/libs/conduit/conduit_schema.hpp b/src/libs/conduit/conduit_schema.hpp index 40240c425..62d0a2d27 100644 --- a/src/libs/conduit/conduit_schema.hpp +++ b/src/libs/conduit/conduit_schema.hpp @@ -195,22 +195,71 @@ class CONDUIT_API Schema //----------------------------------------------------------------------------- void compact_to(Schema &s_dest) const; - std::string to_json(bool detailed=true, - index_t indent=2, + //------------------------------------------------------------------------- + // note: future support: protocol == "yaml" + std::string to_string(const std::string &protocol="json", + index_t indent=2, + index_t depth=0, + const std::string &pad=" ", + const std::string &eoe="\n") const; + + void to_string_stream(std::ostream &os, + const std::string &protocol="json", + index_t indent=2, + index_t depth=0, + const std::string &pad=" ", + const std::string &eoe="\n") const; + + void to_string_stream(const std::string &stream_path, + const std::string &protocol="json", + index_t indent=2, + index_t depth=0, + const std::string &pad=" ", + const std::string &eoe="\n") const; + + // NOTE(cyrush): The primary reason this function exists is to enable easier + // compatibility with debugging tools (e.g. totalview, gdb) that have + // difficulty allocating default string parameters. + std::string to_string_default() const; + + //------------------------------------------------------------------------- + std::string to_json(index_t indent=2, index_t depth=0, const std::string &pad=" ", const std::string &eoe="\n") const; void to_json_stream(std::ostream &os, - bool detailed=true, - index_t indent=2, + index_t indent=2, + index_t depth=0, + const std::string &pad=" ", + const std::string &eoe="\n") const; + + void to_json_stream(const std::string &stream_path, + index_t indent=2, + index_t depth=0, + const std::string &pad=" ", + const std::string &eoe="\n") const; + + //------------------------------------------------------------------------- + ///DEPRECATED: Schema::to_json variants with detailed (bool) arg are + /// deprecated. The detailed arg was never used. These methods + /// will be removed in a future release. + std::string to_json(bool detailed, + index_t indent=2, + index_t depth=0, + const std::string &pad=" ", + const std::string &eoe="\n") const; + + void to_json_stream(std::ostream &os, + bool detailed, + index_t indent=2, index_t depth=0, const std::string &pad=" ", const std::string &eoe="\n") const; void to_json_stream(const std::string &stream_path, - bool detailed=true, - index_t indent=2, + bool detailed, + index_t indent=2, index_t depth=0, const std::string &pad=" ", const std::string &eoe="\n") const; @@ -220,14 +269,25 @@ class CONDUIT_API Schema // difficulty allocating default string parameters. std::string to_json_default() const; + //----------------------------------------------------------------------------- // /// Basic I/O methods // //----------------------------------------------------------------------------- + + ///DEPRECATED: Schema::save variant with detailed (bool) arg is + /// deprecated. The detailed arg was never used. This method + /// will be removed in a future release. + void save(const std::string &ofname, + bool detailed, + index_t indent=2, + index_t depth=0, + const std::string &pad=" ", + const std::string &eoe="\n") const; + void save(const std::string &ofname, - bool detailed=true, - index_t indent=2, + index_t indent=2, index_t depth=0, const std::string &pad=" ", const std::string &eoe="\n") const; diff --git a/src/libs/conduit/python/conduit_python.cpp b/src/libs/conduit/python/conduit_python.cpp index 2bad993c7..e24dbfb32 100644 --- a/src/libs/conduit/python/conduit_python.cpp +++ b/src/libs/conduit/python/conduit_python.cpp @@ -499,8 +499,7 @@ PyConduit_DataType_dealloc(PyConduit_DataType *self) static PyObject * PyConduit_DataType_str(PyConduit_DataType *self) { - std::string output = self->dtype.to_json(); - return (Py_BuildValue("s", output.c_str())); + return (Py_BuildValue("s", self->dtype.to_string().c_str())); } //----------------------------------------------------------------------------- @@ -2522,9 +2521,7 @@ PyConduit_Schema_dealloc(PyConduit_Schema* self) static PyObject * PyConduit_Schema_str(PyConduit_Schema *self) { - std::ostringstream oss; - self->schema->to_json_stream(oss); - return (Py_BuildValue("s", oss.str().c_str())); + return Py_BuildValue("s", self->schema->to_string().c_str()); } @@ -3269,9 +3266,7 @@ PyConduit_NodeIterator_str(PyConduit_NodeIterator *self) { Node n; self->itr.info(n); - std::ostringstream oss; - n.to_json_stream(oss); - return (Py_BuildValue("s", oss.str().c_str())); + return Py_BuildValue("s", n.to_string().c_str()); } //---------------------------------------------------------------------------// @@ -3676,9 +3671,7 @@ PyConduit_Node_dealloc(PyConduit_Node* self) static PyObject * PyConduit_Node_str(PyConduit_Node* self) { - std::ostringstream oss; - self->node->to_json_stream(oss); - return (Py_BuildValue("s", oss.str().c_str())); + return (Py_BuildValue("s", self->node->to_string().c_str())); } //---------------------------------------------------------------------------// @@ -4572,7 +4565,13 @@ PyConduit_Node_info(PyConduit_Node *self) static PyObject * PyConduit_Node_print_detailed(PyConduit_Node *self) { - self->node->print_detailed(); + std::ostringstream oss; + self->node->to_string_stream(oss,"conduit_json"); + // create python string from our c++ stream and call std print + PyObject *py_str = Py_BuildValue("s", oss.str().c_str()); + PyObject_Print(py_str, stdout, Py_PRINT_RAW); + // dec ref for python string + Py_DECREF(py_str); Py_RETURN_NONE; } @@ -4911,6 +4910,80 @@ PyConduit_Node_update_external(PyConduit_Node* self, } +//---------------------------------------------------------------------------// +static PyObject * +PyConduit_Node_to_string(PyConduit_Node* self, + PyObject* args, + PyObject* kwargs) +{ + + Py_ssize_t indent = 2; + Py_ssize_t depth = 0; + + std::string protocol = "yaml"; + std::string pad = " "; + std::string eoe = "\n"; + + char *protocol_c_str = NULL; + char *pad_c_str = NULL; + char *eoe_c_str = NULL; + + static const char *kwlist[] = {"protocol", + "indent", + "depth", + "pad", + "eoe", + NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, + kwargs, + "|snnss", + const_cast(kwlist), + &protocol_c_str, + &indent, + &depth, + &pad_c_str, + &eoe_c_str)) + { + return NULL; + } + + if(protocol_c_str != NULL) + { + protocol = std::string(protocol_c_str); + } + + if(pad_c_str != NULL) + { + pad = std::string(pad_c_str); + } + + if(eoe_c_str != NULL) + { + eoe = std::string(eoe_c_str); + } + + std::ostringstream oss; + + try + { + self->node->to_string_stream(oss, + protocol, + indent, + depth, + pad, + eoe); + } + catch(conduit::Error e) + { + PyErr_SetString(PyExc_IOError, + e.message().c_str()); + return NULL; + } + + return (Py_BuildValue("s", oss.str().c_str())); +} + //---------------------------------------------------------------------------// static PyObject * PyConduit_Node_to_json(PyConduit_Node* self, @@ -5321,15 +5394,24 @@ static PyMethodDef PyConduit_Node_METHODS[] = { METH_NOARGS, "Returns if this node's data is in compact form"}, //-----------------------------------------------------------------------// + {"to_string", + (PyCFunction)PyConduit_Node_to_string, + METH_VARARGS| METH_KEYWORDS, + "Returns a string representation of the node. " + "Optionally takes protocol and spacing options. " + "(Default protocol='yaml'.)"}, + //-----------------------------------------------------------------------// {"to_json", (PyCFunction)PyConduit_Node_to_json, METH_VARARGS| METH_KEYWORDS, - "Returns a JSON string representation of the node."}, + "Returns a JSON string representation of the node. " + "Optionally takes protocol and spacing options."}, //-----------------------------------------------------------------------// {"to_yaml", (PyCFunction)PyConduit_Node_to_yaml, METH_VARARGS| METH_KEYWORDS, - "Returns a YAML string representation of the node."}, + "Returns a YAML string representation of the node. " + "Optionally takes protocol and spacing options."}, //-----------------------------------------------------------------------// {"children", (PyCFunction)PyConduit_Node_iter, diff --git a/src/libs/relay/python/conduit_relay_mpi_python.cpp b/src/libs/relay/python/conduit_relay_mpi_python.cpp index 2a3447f8b..9807ff331 100644 --- a/src/libs/relay/python/conduit_relay_mpi_python.cpp +++ b/src/libs/relay/python/conduit_relay_mpi_python.cpp @@ -611,8 +611,6 @@ PyRelay_MPI_recv_using_schema(PyObject *, //self "comm", NULL}; -std::cout << "PyRelay_MPI_recv_using_schema" << std::endl; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "Onnn", @@ -622,7 +620,6 @@ std::cout << "PyRelay_MPI_recv_using_schema" << std::endl; &tag, &mpi_comm_id)) { - std::cout << "PyRelay_MPI_recv_using_schema ansdsadiosnisaniods" << std::endl; return (NULL); } diff --git a/src/tests/conduit/python/t_python_conduit_node.py b/src/tests/conduit/python/t_python_conduit_node.py index 0fb99f8ae..b00ea0563 100644 --- a/src/tests/conduit/python/t_python_conduit_node.py +++ b/src/tests/conduit/python/t_python_conduit_node.py @@ -65,7 +65,7 @@ def test_simple(self): n['a'] = a_val n['b'] = b_val n['c'] = c_val - + self.assertTrue(n['a'] == a_val) self.assertTrue(n['b'] == b_val) self.assertTrue(n['c'] == c_val) @@ -90,7 +90,7 @@ def test_fetch(self): na = n.fetch('a') na_val = na.value() self.assertEqual(na_val[99], 99) - + def test_child(self): vec = np.array(range(100), np.uint32) n = Node() @@ -100,7 +100,7 @@ def test_child(self): self.assertEqual(na_val[99], 99) n['b'] = vec self.assertEqual(n.number_of_children(),2) - + def test_save_load(self): # on windows, this breaks at 27 !? alen = 26 @@ -113,13 +113,13 @@ def test_save_load(self): nl.load("test_pyconduit_node_save_load.conduit_bin") print(nl) self.assertEqual(nl['a'][alen-1], alen-1) - + n.save("test_pyconduit_node_json_save_load.json",protocol="json") nl = Node() nl.load("test_pyconduit_node_json_save_load.json", protocol="json") print(nl) self.assertEqual(nl['a'][alen-1], alen-1) - + n.save("test_pyconduit_node_base64_json_save_load.conduit_base64_json", protocol="conduit_base64_json") nl = Node() nl.load("test_pyconduit_node_base64_json_save_load.conduit_base64_json", protocol="conduit_base64_json") @@ -297,7 +297,7 @@ def test_diff(self): n2['a'] = 1 self.assertFalse(n1.diff(n2,info)) - + n2['b'] = 2.0 self.assertTrue(n1.diff(n2,info)) self.assertFalse(n1.diff_compatible(n2,info)) @@ -306,7 +306,7 @@ def test_diff(self): def test_list_of_ints(self): # also covered by test_set_all_types - # but this was the reproducer for + # but this was the reproducer for # https://github.com/LLNL/conduit/issues/281 n = Node() a = np.array(list((1,2,3))) @@ -322,7 +322,7 @@ def test_compact_to(self): n['c'] = 3 ni = n.info() self.assertEqual(ni["mem_spaces"].number_of_children(), 3) - + n2 = Node() n.compact_to(n2) ni = n2.info() @@ -333,20 +333,20 @@ def test_update(self): n = Node() data = np.array(range(10), dtype='float64') n["data"].set_external(data) - + print(n) - + n2 = Node() n2.update(n) print(n2) self.assertEqual(n2["data"][0],0) - + n3 = Node() n3.update_external(n) data[0] = 10 print(n3) self.assertEqual(n3["data"][0],10) - + n4 = Node() n4["data"] = 10 n4.update_compatible(n) @@ -358,9 +358,9 @@ def test_reset(self): n = Node() data = np.array(range(10), dtype='float64') n["data"].set_external(data) - + print(n) - + n.reset() self.assertEqual(n.number_of_children(), 0) @@ -383,9 +383,9 @@ def test_child_rename(self): self.assertTrue(n['a'] == a_val) self.assertTrue(n['b'] == b_val) - + n.rename_child('b','c') - + self.assertTrue(n['a'] == a_val) self.assertTrue(n['c'] == b_val) @@ -420,12 +420,12 @@ def test_numeric_tuples(self): self.assertEqual(n['tuple_0'][1], 2) self.assertEqual(n['tuple_0'][2], 3) self.assertEqual(n['tuple_0'][3], 4) - + self.assertEqual(n['tuple_1'][0], 1.0) self.assertEqual(n['tuple_1'][1], 2.0) self.assertEqual(n['tuple_1'][2], 3.0) self.assertEqual(n['tuple_1'][3], 4.0) - + self.assertEqual(n['tuple_2'][0], 1.0) self.assertEqual(n['tuple_2'][1], 2.0) self.assertEqual(n['tuple_2'][2], 3.0) @@ -442,12 +442,12 @@ def test_numeric_lists(self): self.assertEqual(n['list_0'][1], 2) self.assertEqual(n['list_0'][2], 3) self.assertEqual(n['list_0'][3], 4) - + self.assertEqual(n['list_1'][0], 1.0) self.assertEqual(n['list_1'][1], 2.0) self.assertEqual(n['list_1'][2], 3.0) self.assertEqual(n['list_1'][3], 4.0) - + self.assertEqual(n['list_2'][0], 1.0) self.assertEqual(n['list_2'][1], 2.0) self.assertEqual(n['list_2'][2], 3.0) @@ -485,7 +485,7 @@ def test_key_with_slash(self): def test_fetch_existing(self): n = Node() n["my/path"] = 10 - + n_sub = n.fetch_existing("my/path") self.assertEqual(n_sub.value(),10); @@ -493,7 +493,29 @@ def test_fetch_existing(self): with self.assertRaises(Exception): n.fetch_existing('bad/path') + def test_to_string(self): + a_val = np.uint32(10) + b_val = np.uint32(20) + c_val = np.float64(30.0) + + n = Node() + n['a'] = a_val + n['b'] = b_val + n['c'] = c_val + + res_to_str_def = n.to_string() + res_to_str_yaml = n.to_string(protocol="yaml") + res_to_str_json = n.to_string(protocol="json") + + res_to_yaml = n.to_yaml() + res_to_json = n.to_json() + + self.assertEqual(res_to_str_def, res_to_yaml); + self.assertEqual(res_to_str_yaml, res_to_yaml); + + self.assertEqual(res_to_str_json, res_to_json); + n.print_detailed() if __name__ == '__main__': unittest.main() diff --git a/src/tests/conduit/t_conduit_array.cpp b/src/tests/conduit/t_conduit_array.cpp index 69fde04a9..af6689224 100644 --- a/src/tests/conduit/t_conduit_array.cpp +++ b/src/tests/conduit/t_conduit_array.cpp @@ -60,32 +60,36 @@ TEST(conduit_array, basic_construction) { std::vector data1(10,8); std::vector data2(10,-8); - + void *data1_ptr = &data1[0]; const void *cdata2_ptr = &data2[0]; DataArray da_1(data1_ptr,DataType::int8(10)); - + + std::cout << da_1.to_string() << std::endl; + for(index_t i=0;i<10;i++) { EXPECT_EQ(8,da_1[i]); } - + DataArray da_2(cdata2_ptr,DataType::int8(10)); + std::cout << da_2.to_string() << std::endl; + for(index_t i=0;i<10;i++) { EXPECT_EQ(-8,da_2[i]); } - + DataArray da_3(da_1); for(index_t i=0;i<10;i++) { EXPECT_EQ(8,da_3[i]); } - + da_3[0] = 16; - + da_3 = da_2; for(index_t i=0;i<10;i++) @@ -95,6 +99,12 @@ TEST(conduit_array, basic_construction) da_3[0] = -16; + std::cout << da_3.to_string() << std::endl; + + // test other variants of to_string and to stream, etc + da_3.to_string_stream(std::cout); + da_3.to_json_stream(std::cout); + EXPECT_EQ(16,data1[0]); EXPECT_EQ(-16,data2[0]); } diff --git a/src/tests/conduit/t_conduit_datatype_tests.cpp b/src/tests/conduit/t_conduit_datatype_tests.cpp index 8a97d1a17..7756266ac 100644 --- a/src/tests/conduit/t_conduit_datatype_tests.cpp +++ b/src/tests/conduit/t_conduit_datatype_tests.cpp @@ -65,7 +65,7 @@ bool dtypes_equal(const DataType &base, const DataType &ref) { return base.equal //----------------------------------------------------------------------------- void print_dt(const DataType &dtype) { - std::cout << dtype.to_json() << std::endl; + std::cout << dtype.to_string() << std::endl; } //----------------------------------------------------------------------------- diff --git a/src/tests/conduit/t_conduit_node.cpp b/src/tests/conduit/t_conduit_node.cpp index af5ed8507..59ded94e7 100644 --- a/src/tests/conduit/t_conduit_node.cpp +++ b/src/tests/conduit/t_conduit_node.cpp @@ -73,7 +73,7 @@ TEST(conduit_node, simple) EXPECT_EQ(n["c"].as_float64(),c_val); // data should be owned by the conduit Node EXPECT_FALSE(n["a"].is_data_external()); - + } TEST(conduit_node, nested) @@ -351,7 +351,7 @@ TEST(conduit_node, remove_by_index) EXPECT_FALSE(n.has_path("a")); EXPECT_FALSE(n.has_path("b")); EXPECT_FALSE(n.has_path("c")); - + conduit::Generator g2("[{dtype:int64, value: 10},{dtype:int64, value: 20},{dtype:int64, value: 30}]"); conduit::Node n2(g2,true); n2.print(); @@ -360,7 +360,7 @@ TEST(conduit_node, remove_by_index) EXPECT_EQ(n2[0].to_uint64(), 10); EXPECT_EQ(n2[1].to_uint64(), 30); n2.remove(0); - n2.print(); + n2.print(); EXPECT_EQ(n2[0].to_uint64(), 30); } @@ -379,7 +379,7 @@ TEST(conduit_node, check_leaf_assert) TEST(conduit_node, check_value_implict_c_type_cast) { conduit::Node n; - + char cv = 1; short sv = 2; int iv = 3; @@ -407,24 +407,24 @@ TEST(conduit_node, check_value_implict_c_type_cast) n["ssv"] = ssv; n["siv"] = siv; n["slv"] = slv; - + n["ucv"] = ucv; n["usv"] = usv; n["uiv"] = uiv; n["ulv"] = ulv; - + n["fv"] = fv; n["dv"] = dv; - - + + n.print(); - + char cv_r = n["cv"].value(); short sv_r = n["sv"].value(); int iv_r = n["iv"].value(); long lv_r = n["lv"].value(); - - + + EXPECT_EQ(cv,cv_r); EXPECT_EQ(sv,sv_r); EXPECT_EQ(iv,iv_r); @@ -456,13 +456,13 @@ TEST(conduit_node, check_value_implict_c_type_cast) EXPECT_NEAR(fv,fv_r,0.001); EXPECT_NEAR(dv,dv_r,0.001); - - + + char *cv_p_r = n["cv"].value(); short *sv_p_r = n["sv"].value(); int *iv_p_r = n["iv"].value(); long *lv_p_r = n["lv"].value(); - + EXPECT_EQ(cv,cv_p_r[0]); EXPECT_EQ(sv,sv_p_r[0]); EXPECT_EQ(iv,iv_p_r[0]); @@ -495,18 +495,18 @@ TEST(conduit_node, check_value_implict_c_type_cast) EXPECT_NEAR(fv,fv_p_r[0],0.001); EXPECT_NEAR(dv,dv_p_r[0],0.001); - - + + char_array cv_a_r = n["cv"].value(); short_array sv_a_r = n["sv"].value(); int_array iv_a_r = n["iv"].value(); long_array lv_a_r = n["lv"].value(); - + EXPECT_EQ(cv_a_r.number_of_elements(),1); EXPECT_EQ(sv_a_r.number_of_elements(),1); EXPECT_EQ(iv_a_r.number_of_elements(),1); EXPECT_EQ(lv_a_r.number_of_elements(),1); - + EXPECT_EQ(cv,cv_a_r[0]); EXPECT_EQ(sv,sv_a_r[0]); EXPECT_EQ(iv,iv_a_r[0]); @@ -560,12 +560,12 @@ TEST(conduit_node, check_value_implict_c_type_cast) TEST(conduit_node, check_value_implict_bitwidth_type_cast) { conduit::Node n; - + int8 i8v = -1; int16 i16v = -2; int32 i32v = -3; int64 i64v = -4; - + uint8 ui8v = 1; uint16 ui16v = 2; uint32 ui32v = 3; @@ -573,28 +573,28 @@ TEST(conduit_node, check_value_implict_bitwidth_type_cast) float32 f32v = 1.2f; float64 f64v = 2.4; - + n["i8v"] = i8v; n["i16v"] = i16v; n["i32v"] = i32v; n["i64v"] = i64v; - + n["ui8v"] = ui8v; n["ui16v"] = ui16v; n["ui32v"] = ui32v; n["ui64v"] = ui64v; - + n["f32v"] = f32v; n["f64v"] = f64v; - - + + n.print(); - + int8 i8v_r = n["i8v"].value(); int16 i16v_r = n["i16v"].value(); int32 i32v_r = n["i32v"].value(); int64 i64v_r = n["i64v"].value(); - + EXPECT_EQ(i8v,i8v_r); EXPECT_EQ(i16v,i16v_r); EXPECT_EQ(i32v,i32v_r); @@ -604,7 +604,7 @@ TEST(conduit_node, check_value_implict_bitwidth_type_cast) uint16 ui16v_r = n["ui16v"].value(); uint32 ui32v_r = n["ui32v"].value(); uint64 ui64v_r = n["ui64v"].value(); - + EXPECT_EQ(ui8v,ui8v_r); EXPECT_EQ(ui16v,ui16v_r); EXPECT_EQ(ui32v,ui32v_r); @@ -616,13 +616,13 @@ TEST(conduit_node, check_value_implict_bitwidth_type_cast) EXPECT_NEAR(f32v,f32v_r,0.001); EXPECT_NEAR(f64v,f64v_r,0.001); - - + + int8 *i8v_p_r = n["i8v"].value(); int16 *i16v_p_r = n["i16v"].value(); int32 *i32v_p_r = n["i32v"].value(); int64 *i64v_p_r = n["i64v"].value(); - + EXPECT_EQ(i8v,i8v_p_r[0]); EXPECT_EQ(i16v,i16v_p_r[0]); EXPECT_EQ(i32v,i32v_p_r[0]); @@ -632,7 +632,7 @@ TEST(conduit_node, check_value_implict_bitwidth_type_cast) uint16 *ui16v_p_r = n["ui16v"].value(); uint32 *ui32v_p_r = n["ui32v"].value(); uint64 *ui64v_p_r = n["ui64v"].value(); - + EXPECT_EQ(ui8v,ui8v_p_r[0]); EXPECT_EQ(ui16v,ui16v_p_r[0]); EXPECT_EQ(ui32v,ui32v_p_r[0]); @@ -644,7 +644,7 @@ TEST(conduit_node, check_value_implict_bitwidth_type_cast) EXPECT_NEAR(f32v,f32v_p_r[0],0.001); EXPECT_NEAR(f64v,f64v_p_r[0],0.001); - + int8_array i8v_a_r = n["i8v"].value(); int16_array i16v_a_r = n["i16v"].value(); int32_array i32v_a_r = n["i32v"].value(); @@ -654,7 +654,7 @@ TEST(conduit_node, check_value_implict_bitwidth_type_cast) EXPECT_EQ(i16v_a_r.number_of_elements(),1); EXPECT_EQ(i32v_a_r.number_of_elements(),1); EXPECT_EQ(i64v_a_r.number_of_elements(),1); - + EXPECT_EQ(i8v,i8v_a_r[0]); EXPECT_EQ(i16v,i16v_a_r[0]); EXPECT_EQ(i32v,i32v_a_r[0]); @@ -664,12 +664,12 @@ TEST(conduit_node, check_value_implict_bitwidth_type_cast) uint16_array ui16v_a_r = n["ui16v"].value(); uint32_array ui32v_a_r = n["ui32v"].value(); uint64_array ui64v_a_r = n["ui64v"].value(); - + EXPECT_EQ(ui8v_a_r.number_of_elements(),1); EXPECT_EQ(ui16v_a_r.number_of_elements(),1); EXPECT_EQ(ui32v_a_r.number_of_elements(),1); EXPECT_EQ(ui64v_a_r.number_of_elements(),1); - + EXPECT_EQ(ui8v,ui8v_a_r[0]); EXPECT_EQ(ui16v,ui16v_a_r[0]); EXPECT_EQ(ui32v,ui32v_a_r[0]); @@ -684,13 +684,13 @@ TEST(conduit_node, check_value_implict_bitwidth_type_cast) EXPECT_NEAR(f32v,f32v_a_r[0],0.001); EXPECT_NEAR(f64v,f64v_a_r[0],0.001); - - + + } //----------------------------------------------------------------------------- -void +void print_warning(const std::string &msg, const std::string &file, int line) @@ -708,31 +708,31 @@ TEST(conduit_node, check_as_value_default_after_warning) uint64 val = 10; Node n; n.set(val); - + EXPECT_EQ(n.as_uint8(),0); EXPECT_EQ(n.as_uint8_ptr(),(void*)NULL); - + float32_array arr = n.value(); - EXPECT_TRUE(arr.dtype().is_empty()); + EXPECT_TRUE(arr.dtype().is_empty()); EXPECT_EQ(arr.data_ptr(),(void*)NULL); - + conduit::utils::set_warning_handler(conduit::utils::default_warning_handler); } //----------------------------------------------------------------------------- TEST(conduit_node, check_contiguous) -{ +{ uint8 u8av[6] = {2,4,8,16,32,64}; uint16 u16av[6] = {2,4,8,16,32,64}; uint32 u32av[6] = {2,4,8,16,32,64}; uint64 u64av[6] = {2,4,8,16,32,64}; - + uint8_array u8av_a(u8av,DataType::uint8(6)); uint16_array u16av_a(u16av,DataType::uint16(6)); uint32_array u32av_a(u32av,DataType::uint32(6)); uint64_array u64av_a(u64av,DataType::uint64(6)); - + Node n; n["a"] = u8av_a; n["b"] = u16av_a; @@ -743,21 +743,21 @@ TEST(conduit_node, check_contiguous) EXPECT_TRUE(n.is_compact()); // but not contig EXPECT_FALSE(n.is_contiguous()); - // contig dptr should be null if not contig + // contig dptr should be null if not contig EXPECT_FALSE(n.contiguous_data_ptr() != NULL); - + // compact to create compact + contig Node n2; n.compact_to(n2); EXPECT_TRUE(n2.is_compact()); EXPECT_TRUE(n2.is_contiguous()); - // contig dptr should not be null if contig + // contig dptr should not be null if contig EXPECT_TRUE(n2.contiguous_data_ptr() != NULL); // no longer contig n2["e"] = 10; EXPECT_FALSE(n2.is_contiguous()); - // contig dptr should be null if not contig + // contig dptr should be null if not contig EXPECT_FALSE(n2.contiguous_data_ptr() != NULL); // still compact EXPECT_TRUE(n2.is_compact()); @@ -767,37 +767,37 @@ TEST(conduit_node, check_contiguous) n3["a"].set_external(u64av,2); n3["b"].set_external(u64av,4,sizeof(uint64)*2); EXPECT_TRUE(n3.is_contiguous()); - // contig dptr should not be null if contig + // contig dptr should not be null if contig EXPECT_TRUE(n3.contiguous_data_ptr() != NULL); - - + + // make non contig n3["c"].set_external(u64av,3,sizeof(uint64)*3); EXPECT_FALSE(n3.is_contiguous()); - // contig dptr should be null if not contig + // contig dptr should be null if not contig EXPECT_TRUE(n3.contiguous_data_ptr() == NULL); - - + + // contig but not compact Node n4; n4["a"].set_external(u64av,2); n4["b"].set_external(u64av,2,sizeof(uint64)*2,sizeof(uint64)*2); EXPECT_FALSE(n4.is_compact()); EXPECT_TRUE(n4.is_contiguous()); - - + + // nested contig and compact Node n5; n5["a/b/c/d/e/f"].set_int64(10); - + EXPECT_TRUE(n5.is_compact()); EXPECT_TRUE(n5.is_contiguous()); - + void *n5_contg_ptr = n5.contiguous_data_ptr(); - - // contig dptr should be null if not contig + + // contig dptr should be null if not contig EXPECT_TRUE(n5_contg_ptr != NULL); - + // check loc and value of contig dptr EXPECT_EQ(n5_contg_ptr, @@ -814,9 +814,9 @@ TEST(conduit_node, check_contiguous) //----------------------------------------------------------------------------- TEST(conduit_node, check_contiguous_with) -{ +{ uint64 u64av[10] = {2,4,8,16,32,64,128,256,512,1024}; - + Node n1; n1["a"].set_external(u64av,5); n1["b"].set_external(u64av,5,5 * sizeof(uint64)); @@ -831,42 +831,42 @@ TEST(conduit_node, check_contiguous_with) // we don't expect things to be contig with NULL EXPECT_FALSE(n1["a"].contiguous_with(NULL)); EXPECT_FALSE(n1["b"].contiguous_with(NULL)); - + // b should be contig with a EXPECT_TRUE(n1["b"].contiguous_with(n1["a"])); - + // but the reverse is not the case (b comes after a ...) EXPECT_FALSE(n1["a"].contiguous_with(n1["b"])); - + // b it should be contig with address at the end of a // a.ele_ptr(5) should land us right at start of b EXPECT_EQ(n1["b"].element_ptr(0),n1["a"].element_ptr(5)); - + // b it should be contig with address at the end of a EXPECT_TRUE(n1["b"].contiguous_with(n1["a"].element_ptr(5))); - - + + Node n2; n2["a"].set_external(u64av,5); n2["b"].set_external(DataType::uint8(),NULL); n2["c"].set_external(u64av,5,5 * sizeof(uint64)); - + // we expect c to be contig with a EXPECT_TRUE(n1["c"].contiguous_with(n1["a"])); - + // null leaf type in middle should break contig EXPECT_FALSE(n2.is_contiguous()); - + // should be contig if we removed the null leaf n2.remove("b"); EXPECT_TRUE(n2.is_contiguous()); - + // but an empy leaf type in middle shouldn't break contig n2["a"].set_external(u64av,5); n2["b"].set(DataType::empty()); n2["c"].set_external(u64av,5,5 * sizeof(uint64)); - + EXPECT_TRUE(n2.is_contiguous()); } @@ -874,18 +874,18 @@ TEST(conduit_node, check_contiguous_with) //----------------------------------------------------------------------------- TEST(conduit_node, check_path) -{ +{ Node n; - + n["a/b/c/d/e/f"] = 10; EXPECT_EQ(n.path(),""); EXPECT_EQ(n["a/b/c/d/e/f"].path(), "a/b/c/d/e/f"); - + // check roundtrip -- using path() to fetch from root node EXPECT_EQ(n.fetch_ptr(n["a/b/c/d/e/f"].path()),&n["a/b/c/d/e/f"]); - + // list cases EXPECT_EQ(n["a/b/c/list"].append().path(), "a/b/c/list/[0]"); EXPECT_EQ(n["a/b/c/list"].append().path(), "a/b/c/list/[1]"); @@ -898,14 +898,14 @@ TEST(conduit_node, check_path) //----------------------------------------------------------------------------- TEST(conduit_node, check_path_in_bad_access) -{ +{ // test if the path string appears in the exception thrown // for a bad dtype access - + Node n; - + n["a/b/c/d/e/f"] = 10; - + try { std::string s = n["a/b/c/d/e/f"].as_string(); @@ -919,12 +919,12 @@ TEST(conduit_node, check_path_in_bad_access) //----------------------------------------------------------------------------- TEST(conduit_node, check_const_access) -{ +{ Node n; - + n["a/b"].set_int32(10); n["a/c"].set(DataType::int64(2)); - + int64_array c_vals = n["a/c"].value(); c_vals[0]= 1; c_vals[1]= 2; @@ -932,7 +932,7 @@ TEST(conduit_node, check_const_access) // Note: this won't throw b/c n is not const, so the const fetch // will not bind //const Node &n_bad = n["bad"]; - + const Node &n_a = n["a"]; const int64 *c_vals_const = n_a["c"].as_int64_ptr(); @@ -940,23 +940,23 @@ TEST(conduit_node, check_const_access) EXPECT_THROW(const Node &n_a_bad = n_a.fetch_existing("bad");,conduit::Error); EXPECT_THROW(const Node &n_a_bad = n_a["bad"];,conduit::Error); - + EXPECT_EQ(n_a["b"].as_int32(),10); EXPECT_EQ(c_vals_const[0],1); EXPECT_EQ(c_vals_const[1],2); - + } //----------------------------------------------------------------------------- TEST(conduit_node, node_more_set_cases) -{ +{ // construct from data type Node n(DataType::object()); - + // create complex tree n["a/b/c/d"].append().set(DataType::int64(2)); - + int64_array cvals = n["a/b/c/d"][0].value(); cvals[0]= 1; cvals[1]= 2; @@ -974,10 +974,10 @@ TEST(conduit_node, node_more_set_cases) EXPECT_EQ(n2_vals_ptr[0],1); EXPECT_EQ(n2_vals_ptr[1],2); - + EXPECT_EQ(n3_vals_ptr[0],1); EXPECT_EQ(n3_vals_ptr[1],2); - + float64 fval[1] = { 3.1415 }; @@ -996,7 +996,7 @@ TEST(conduit_node, node_more_set_cases) EXPECT_EQ(n5["a"].as_float64(),fval[0]); EXPECT_EQ(n5["b"].as_float64(),fval[0]); - + n5.print(); @@ -1005,19 +1005,19 @@ TEST(conduit_node, node_more_set_cases) //----------------------------------------------------------------------------- TEST(conduit_node, check_invalid_index_error) -{ +{ Node n; - + EXPECT_THROW(n.child(0),conduit::Error); EXPECT_THROW(n[0],conduit::Error); EXPECT_THROW(n.child(1),conduit::Error); EXPECT_THROW(n[1],conduit::Error); - + n.append(); - + Node &n_0 = n[0]; - - + + } @@ -1032,20 +1032,20 @@ TEST(conduit_node, check_empty_path_fetch_error) TEST(conduit_node, rename_child) { Node n; - + // error, can't rename non object EXPECT_THROW(n.rename_child("a","b"),conduit::Error); - + n["a"].set(DataType::int64()); n["b"].set(DataType::float64()); n["c"].set(DataType::float32(10)); - + float32_array c_vals = n["c"].value(); for(index_t i=0;i<10;i++) { c_vals[i] = i; } - + n.print(); // error, can't rename to existing child name @@ -1094,7 +1094,7 @@ TEST(conduit_node, list_to_obj_cleanup) //----------------------------------------------------------------------------- TEST(conduit_node, test_parse_all_protos) -{ +{ Node n; n["a/b/c"] = (int64) 10; @@ -1111,22 +1111,84 @@ TEST(conduit_node, test_parse_all_protos) n2.parse(txt_cases[0],"json"); EXPECT_FALSE(n.diff(n2,info)); info.print(); - + n2.parse(txt_cases[1],"conduit_json"); EXPECT_FALSE(n.diff(n2,info)); - + n2.parse(txt_cases[2],"conduit_base64_json"); EXPECT_FALSE(n.diff(n2,info)); + + n2.parse(txt_cases[3],"yaml"); + EXPECT_FALSE(n.diff(n2,info)); + +} + + +//----------------------------------------------------------------------------- +TEST(conduit_node, to_string_and_parse_all_protos) +{ + Node n; + + n["a/b/c"] = (int64) 10; + n["a/b/d"] = (float64) 42.2; + n["a/b/e"] = " string !"; + + std::ostringstream oss; + std::vector txt_cases; + txt_cases.push_back(n.to_string()); // yaml + txt_cases.push_back(n.to_string_default()); // yaml + + n.to_string_stream(oss); + txt_cases.push_back(oss.str()); // yaml + + txt_cases.push_back(n.to_string("yaml")); + + oss.str(""); + n.to_string_stream(oss,"json"); + txt_cases.push_back(oss.str()); // json + + txt_cases.push_back(n.to_string("json")); + txt_cases.push_back(n.to_string("conduit_json")); + txt_cases.push_back(n.to_string("conduit_base64_json")); + + Node n2, info; + + n2.parse(txt_cases[0],"yaml"); + EXPECT_FALSE(n.diff(n2,info)); + info.print(); + + n2.parse(txt_cases[1],"yaml"); + EXPECT_FALSE(n.diff(n2,info)); + info.print(); + + n2.parse(txt_cases[2],"yaml"); + EXPECT_FALSE(n.diff(n2,info)); + info.print(); + n2.parse(txt_cases[3],"yaml"); EXPECT_FALSE(n.diff(n2,info)); + info.print(); + + n2.parse(txt_cases[4],"json"); + EXPECT_FALSE(n.diff(n2,info)); + info.print(); + + n2.parse(txt_cases[5],"json"); + EXPECT_FALSE(n.diff(n2,info)); + info.print(); + n2.parse(txt_cases[6],"conduit_json"); + EXPECT_FALSE(n.diff(n2,info)); + + n2.parse(txt_cases[7],"conduit_base64_json"); + EXPECT_FALSE(n.diff(n2,info)); } //----------------------------------------------------------------------------- TEST(conduit_node, add_child) -{ +{ Node n; int64 val = 42; Node &n_a = n.add_child("a"); @@ -1138,3 +1200,5 @@ TEST(conduit_node, add_child) EXPECT_EQ(&n_a,n_ptr); } + + diff --git a/src/tests/conduit/t_conduit_schema.cpp b/src/tests/conduit/t_conduit_schema.cpp index 8090ec1f9..4c969b434 100644 --- a/src/tests/conduit/t_conduit_schema.cpp +++ b/src/tests/conduit/t_conduit_schema.cpp @@ -394,6 +394,58 @@ TEST(schema_basics, pathlike_child_names) } +//----------------------------------------------------------------------------- +TEST(schema_basics, schema_to_string) +{ + Schema s; + s["a"].set(DataType::int64()); + s["b"].set(DataType::float64()); + s["c"].set(DataType::float64()); + + std::string res_str = s.to_string(); + std::string res_json = s.to_json(); + + std::ostringstream oss; + + s.to_string_stream(oss); + std::string res_str_from_oss = oss.str(); + + oss.str(""); + s.to_json_stream(oss); + std::string res_json_from_oss = oss.str(); + + // save files + std::string tf_t_str_file ="tout_schema_to_string_stream_file.json"; + // remove if exists + if(utils::is_file(tf_t_str_file)) + { + utils::remove_file(tf_t_str_file); + } + s.to_string_stream(tf_t_str_file); + EXPECT_TRUE(utils::is_file(tf_t_str_file)); + + std::string tf_t_json_file ="tout_schema_to_string_stream_file.json"; + // remove if exists + if(utils::is_file(tf_t_json_file)) + { + utils::remove_file(tf_t_str_file); + } + + s.to_json_stream(tf_t_json_file); + + EXPECT_TRUE(utils::is_file(tf_t_json_file)); + + std::cout << res_str << std::endl; + std::cout << res_json << std::endl; + std::cout << res_str_from_oss << std::endl; + std::cout << res_json_from_oss << std::endl; + + // we expect these to be the same + EXPECT_EQ(res_str, res_json); + EXPECT_EQ(res_str, res_str_from_oss); + EXPECT_EQ(res_str, res_json_from_oss); +} + //----------------------------------------------------------------------------- /// diff --git a/src/tests/conduit/t_conduit_yaml.cpp b/src/tests/conduit/t_conduit_yaml.cpp index 81e092575..e192fe111 100644 --- a/src/tests/conduit/t_conduit_yaml.cpp +++ b/src/tests/conduit/t_conduit_yaml.cpp @@ -628,3 +628,21 @@ TEST(conduit_yaml, dup_object_name_error) ASSERT_THROW(g3.walk(n),conduit::Error); EXPECT_TRUE(n3.dtype().is_empty()); } + + +//----------------------------------------------------------------------------- +TEST(conduit_yaml, to_yaml_leaf_nl) +{ + uint32 a_val = 10; + + Node n; + n = a_val; + + std::string res = n.to_yaml(); + std::cout << res << std::endl; + // we don't want leaf to end with nl + // this check probes that + EXPECT_NE(res[2],std::string("\n")[0]); + +} + diff --git a/src/tests/docs/t_conduit_docs_blueprint_demos.cpp b/src/tests/docs/t_conduit_docs_blueprint_demos.cpp index 6da2461d1..92496534c 100644 --- a/src/tests/docs/t_conduit_docs_blueprint_demos.cpp +++ b/src/tests/docs/t_conduit_docs_blueprint_demos.cpp @@ -51,6 +51,7 @@ #include "conduit.hpp" #include "conduit_blueprint.hpp" #include "conduit_relay.hpp" +#include "t_conduit_docs_tutorial_helpers.hpp" #include #include "gtest/gtest.h" @@ -135,12 +136,14 @@ void validate_basic_example(const std::string &name, //----------------------------------------------------------------------------- TEST(conduit_docs, blueprint_demo_basic_uniform) { + BEGIN_EXAMPLE("blueprint_demo_basic_uniform"); // create container node Node mesh; // generate simple uniform 2d 'basic' mesh conduit::blueprint::mesh::examples::basic("uniform", 3, 3, 0, mesh); // print out results mesh.print(); + END_EXAMPLE("blueprint_demo_basic_uniform"); const std::string mesh_json = R"( { @@ -193,12 +196,14 @@ TEST(conduit_docs, blueprint_demo_basic_uniform) //----------------------------------------------------------------------------- TEST(conduit_docs, blueprint_demo_basic_rectilinear) { + BEGIN_EXAMPLE("blueprint_demo_basic_rectilinear"); // create container node Node mesh; // generate simple rectilinear 2d 'basic' mesh conduit::blueprint::mesh::examples::basic("rectilinear", 3, 3, 0, mesh); // print out results mesh.print(); + END_EXAMPLE("blueprint_demo_basic_rectilinear"); const std::string mesh_json = R"( { @@ -241,12 +246,14 @@ TEST(conduit_docs, blueprint_demo_basic_rectilinear) //----------------------------------------------------------------------------- TEST(conduit_docs, blueprint_demo_basic_structured) { + BEGIN_EXAMPLE("blueprint_demo_basic_structured"); // create container node Node mesh; // generate simple structured 2d 'basic' mesh conduit::blueprint::mesh::examples::basic("structured", 3, 3, 1, mesh); // print out results mesh.print(); + END_EXAMPLE("blueprint_demo_basic_structured"); const std::string mesh_json = R"( { @@ -297,12 +304,14 @@ TEST(conduit_docs, blueprint_demo_basic_structured) //----------------------------------------------------------------------------- TEST(conduit_docs, blueprint_demo_basic_tris) { + BEGIN_EXAMPLE("blueprint_demo_basic_tris"); // create container node Node mesh; // generate simple explicit tri-based 2d 'basic' mesh conduit::blueprint::mesh::examples::basic("tris", 3, 3, 0, mesh); // print out results mesh.print(); + END_EXAMPLE("blueprint_demo_basic_tris"); const std::string mesh_json = R"( { @@ -350,12 +359,14 @@ TEST(conduit_docs, blueprint_demo_basic_tris) //----------------------------------------------------------------------------- TEST(conduit_docs, blueprint_demo_basic_quads) { + BEGIN_EXAMPLE("blueprint_demo_basic_quads"); // create container node Node mesh; // generate simple explicit quad-based 2d 'basic' mesh conduit::blueprint::mesh::examples::basic("quads", 3, 3, 0, mesh); // print out results mesh.print(); + END_EXAMPLE("blueprint_demo_basic_quads"); const std::string mesh_json = R"( { @@ -403,13 +414,15 @@ TEST(conduit_docs, blueprint_demo_basic_quads) //----------------------------------------------------------------------------- TEST(conduit_docs, blueprint_demo_basic_tets) { + BEGIN_EXAMPLE("blueprint_demo_basic_tets"); // create container node Node mesh; // generate simple explicit tri-based 3d 'basic' mesh conduit::blueprint::mesh::examples::basic("tets", 3, 3, 3, mesh); // print out results mesh.print(); - + END_EXAMPLE("blueprint_demo_basic_tets"); + const std::string mesh_json = R"( { "coordsets": @@ -457,12 +470,14 @@ TEST(conduit_docs, blueprint_demo_basic_tets) //----------------------------------------------------------------------------- TEST(conduit_docs, blueprint_demo_basic_hexs) { + BEGIN_EXAMPLE("blueprint_demo_basic_hexs"); // create container node Node mesh; // generate simple explicit quad-based 3d 'basic' mesh conduit::blueprint::mesh::examples::basic("hexs", 3, 3, 3, mesh); // print out results mesh.print(); + END_EXAMPLE("blueprint_demo_basic_hexs"); const std::string mesh_json = R"( { @@ -511,12 +526,14 @@ TEST(conduit_docs, blueprint_demo_basic_hexs) //----------------------------------------------------------------------------- TEST(conduit_docs, blueprint_demo_basic_polygons) { + BEGIN_EXAMPLE("blueprint_demo_basic_polygons"); // create container node Node mesh; // generate simple explicit poly-based 2d 'basic' mesh conduit::blueprint::mesh::examples::basic("polygons", 3, 3, 0, mesh); // print out results mesh.print(); + END_EXAMPLE("blueprint_demo_basic_polygons"); const std::string mesh_json = R"( { @@ -564,12 +581,14 @@ TEST(conduit_docs, blueprint_demo_basic_polygons) //----------------------------------------------------------------------------- TEST(conduit_docs, blueprint_demo_basic_polyhedra) { + BEGIN_EXAMPLE("blueprint_demo_basic_polyhedra"); // create container node Node mesh; // generate simple explicit poly-based 3d 'basic' mesh conduit::blueprint::mesh::examples::basic("polyhedra", 3, 3, 3, mesh); // print out results mesh.print(); + END_EXAMPLE("blueprint_demo_basic_polyhedra"); const std::string mesh_json = R"( { @@ -618,6 +637,7 @@ TEST(conduit_docs, blueprint_demo_basic_polyhedra) //----------------------------------------------------------------------------- TEST(conduit_docs, blueprint_demo_basic_uniform_detailed) { + BEGIN_EXAMPLE("blueprint_demo_basic_uniform_detailed"); // create a Conduit node to hold our mesh data Node mesh; @@ -678,4 +698,5 @@ TEST(conduit_docs, blueprint_demo_basic_uniform_detailed) // save our mesh to a json that can be read by VisIt conduit::relay::io_blueprint::save(mesh, "basic_detailed_uniform.blueprint_root"); + END_EXAMPLE("blueprint_demo_basic_uniform_detailed"); } diff --git a/src/tests/docs/t_conduit_docs_blueprint_examples.cpp b/src/tests/docs/t_conduit_docs_blueprint_examples.cpp index 23c5d2567..3f8be54d4 100644 --- a/src/tests/docs/t_conduit_docs_blueprint_examples.cpp +++ b/src/tests/docs/t_conduit_docs_blueprint_examples.cpp @@ -51,17 +51,16 @@ #include "conduit.hpp" #include "conduit_blueprint.hpp" #include "conduit_relay.hpp" +#include "t_conduit_docs_tutorial_helpers.hpp" #include #include "gtest/gtest.h" using namespace conduit; //----------------------------------------------------------------------------- -// 65-79 TEST(conduit_docs, blueprint_example_1) { - CONDUIT_INFO("blueprint_example_1"); - + BEGIN_EXAMPLE("blueprint_example_1"); // setup our candidate and info nodes Node n, info; @@ -74,25 +73,22 @@ TEST(conduit_docs, blueprint_example_1) std::cout << "mesh verify succeeded." << std::endl; else std::cout << "mesh verify failed!" << std::endl; - + // show some of the verify details info["coordsets"].print(); - - CONDUIT_INFO("blueprint_example_1"); + END_EXAMPLE("blueprint_example_1"); } //----------------------------------------------------------------------------- -// 90-110 TEST(conduit_docs, blueprint_example_2) { - CONDUIT_INFO("blueprint_example_2"); - + BEGIN_EXAMPLE("blueprint_example_2"); // setup our candidate and info nodes Node n, verify_info, mem_info; // create an example mcarray conduit::blueprint::mcarray::examples::xyz("separate",5,n); - + std::cout << "example 'separate' mcarray " << std::endl; n.print(); n.info(mem_info); @@ -113,6 +109,5 @@ TEST(conduit_docs, blueprint_example_2) mem_info.print(); } } - - CONDUIT_INFO("blueprint_example_2"); + END_EXAMPLE("blueprint_example_2"); } diff --git a/src/tests/docs/t_conduit_docs_relay_io_generic_examples.cpp b/src/tests/docs/t_conduit_docs_relay_io_generic_examples.cpp index 362b56e9f..20ab04dcf 100644 --- a/src/tests/docs/t_conduit_docs_relay_io_generic_examples.cpp +++ b/src/tests/docs/t_conduit_docs_relay_io_generic_examples.cpp @@ -51,17 +51,16 @@ #include "conduit.hpp" #include "conduit_blueprint.hpp" #include "conduit_relay.hpp" +#include "t_conduit_docs_tutorial_helpers.hpp" #include #include "gtest/gtest.h" using namespace conduit; //----------------------------------------------------------------------------- -// 65-78 TEST(conduit_docs, relay_io_example_1_json) { - CONDUIT_INFO("relay_io_example_1_json"); - + BEGIN_EXAMPLE("relay_io_example_1_json"); // setup node to save Node n; n["a/my_data"] = 1.0; @@ -77,54 +76,48 @@ TEST(conduit_docs, relay_io_example_1_json) conduit::relay::io::load("my_output.json",n_load); std::cout << "\nLoad result:" << std::endl; n_load.print(); - - CONDUIT_INFO("relay_io_example_1_json"); + END_EXAMPLE("relay_io_example_1_json"); } #ifdef CONDUIT_RELAY_IO_HDF5_ENABLED //----------------------------------------------------------------------------- -// 91-107 TEST(conduit_docs, relay_io_example_1_hdf5) { - CONDUIT_INFO("relay_io_example_1_hdf5"); - + BEGIN_EXAMPLE("relay_io_example_1_hdf5"); // setup node to save Node n; n["a/my_data"] = 1.0; n["a/b/my_string"] = "value"; std::cout << "\nNode to write:" << std::endl; n.print(); - + //save to hdf5 using save conduit::relay::io::save(n,"my_output.hdf5"); - + //load back from hdf5 using load Node n_load; conduit::relay::io::load("my_output.hdf5",n_load); std::cout << "\nLoad result:" << std::endl; n_load.print(); - - CONDUIT_INFO("relay_io_example_1_hdf5"); + END_EXAMPLE("relay_io_example_1_hdf5"); } //----------------------------------------------------------------------------- -// 118-140 TEST(conduit_docs, relay_io_example_2_hdf5) { - CONDUIT_INFO("relay_io_example_2_hdf5"); - + BEGIN_EXAMPLE("relay_io_example_2_hdf5"); // setup node to save Node n; n["a/my_data"] = 1.0; n["a/b/my_string"] = "value"; std::cout << "\nNode to write:" << std::endl; n.print(); - + //save to hdf5 using save conduit::relay::io::save(n,"my_output.hdf5"); - + // append a new path to the hdf5 file using save_merged Node n2; n2["a/b/new_data"] = 42.0; @@ -137,26 +130,23 @@ TEST(conduit_docs, relay_io_example_2_hdf5) conduit::relay::io::load("my_output.hdf5",n_load); std::cout << "\nLoad result:" << std::endl; n_load.print(); - - CONDUIT_INFO("relay_io_example_2_hdf5"); + END_EXAMPLE("relay_io_example_2_hdf5"); } //----------------------------------------------------------------------------- -// 150-168 TEST(conduit_docs, relay_io_example_3_hdf5) { - CONDUIT_INFO("relay_io_example_3_hdf5"); - + BEGIN_EXAMPLE("relay_io_example_3_hdf5"); // setup node to save Node n; n["a/my_data"] = 1.0; n["a/b/my_string"] = "value"; std::cout << "\nNode to write:" << std::endl; n.print(); - + //save to hdf5 using generic i/o save conduit::relay::io::save(n,"my_output.hdf5"); - + // append to existing node with data from hdf5 file using load_merged Node n_load; n_load["a/b/new_data"] = 42.0; @@ -165,57 +155,49 @@ TEST(conduit_docs, relay_io_example_3_hdf5) conduit::relay::io::load_merged("my_output.hdf5",n_load); std::cout << "\nLoad result:" << std::endl; n_load.print(); - - CONDUIT_INFO("relay_io_example_3_hdf5"); + END_EXAMPLE("relay_io_example_3_hdf5"); } - //----------------------------------------------------------------------------- -// 179-193 TEST(conduit_docs, relay_io_example_4_hdf5) { - CONDUIT_INFO("relay_io_example_4_hdf5"); - + BEGIN_EXAMPLE("relay_io_example_4_hdf5"); // setup node to save Node n; n["path/to/my_data"] = 1.0; std::cout << "\nNode to write:" << std::endl; n.print(); - + //save to hdf5 using generic i/o save conduit::relay::io::save(n,"my_output.hdf5"); - + // load only a subset of the tree Node n_load; conduit::relay::io::load("my_output.hdf5:path/to",n_load); std::cout << "\nLoad result from 'path/to'" << std::endl; n_load.print(); - - CONDUIT_INFO("relay_io_example_4_hdf5"); + END_EXAMPLE("relay_io_example_4_hdf5"); } //----------------------------------------------------------------------------- -// 203-217 TEST(conduit_docs, relay_io_example_5_hdf5) { - CONDUIT_INFO("relay_io_example_5_hdf5"); - + BEGIN_EXAMPLE("relay_io_example_5_hdf5"); // setup node to save Node n; n["my_data"] = 1.0; std::cout << "\nNode to write to 'path/to':" << std::endl; n.print(); - + //save to hdf5 using generic i/o save conduit::relay::io::save(n,"my_output.hdf5:path/to"); - + // load only a subset of the tree Node n_load; conduit::relay::io::load("my_output.hdf5",n_load); std::cout << "\nLoad result:" << std::endl; n_load.print(); - - CONDUIT_INFO("relay_io_example_4_hdf5"); + END_EXAMPLE("relay_io_example_5_hdf5"); } diff --git a/src/tests/docs/t_conduit_docs_relay_io_handle_examples.cpp b/src/tests/docs/t_conduit_docs_relay_io_handle_examples.cpp index 5e16e74c5..238a81590 100644 --- a/src/tests/docs/t_conduit_docs_relay_io_handle_examples.cpp +++ b/src/tests/docs/t_conduit_docs_relay_io_handle_examples.cpp @@ -51,6 +51,7 @@ #include "conduit.hpp" #include "conduit_blueprint.hpp" #include "conduit_relay.hpp" +#include "t_conduit_docs_tutorial_helpers.hpp" #include #include "gtest/gtest.h" @@ -60,8 +61,7 @@ using namespace conduit; // 65-114 TEST(conduit_docs, relay_io_handle_1) { - CONDUIT_INFO("relay_io_handle_example_1"); - + BEGIN_EXAMPLE("relay_io_handle_example_1"); // setup node with example data to save Node n; n["a/data"] = 1.0; @@ -130,7 +130,6 @@ TEST(conduit_docs, relay_io_handle_1) std::cout << "\nRead Result:" << std::endl; nread.print(); - - CONDUIT_INFO("relay_io_handle_example_1"); + END_EXAMPLE("relay_io_handle_example_1"); } diff --git a/src/tests/docs/t_conduit_docs_relay_io_hdf5_examples.cpp b/src/tests/docs/t_conduit_docs_relay_io_hdf5_examples.cpp index f3012c3ab..d21832a2f 100644 --- a/src/tests/docs/t_conduit_docs_relay_io_hdf5_examples.cpp +++ b/src/tests/docs/t_conduit_docs_relay_io_hdf5_examples.cpp @@ -52,17 +52,16 @@ #include "conduit_blueprint.hpp" #include "conduit_relay.hpp" #include "conduit_relay_io_hdf5.hpp" +#include "t_conduit_docs_tutorial_helpers.hpp" #include #include "gtest/gtest.h" using namespace conduit; //----------------------------------------------------------------------------- -// 91-107 TEST(conduit_docs, relay_io_example_hdf5_interface_1) { - CONDUIT_INFO("relay_io_example_hdf5_interface_1"); - + BEGIN_EXAMPLE("relay_io_example_hdf5_interface_1"); // setup node to save Node n; n["a/my_data"] = 1.0; @@ -113,23 +112,20 @@ TEST(conduit_docs, relay_io_example_hdf5_interface_1) if(conduit::relay::io::is_hdf5_file("myoutput.hdf5")) std::cout << "File \n'myoutput.hdf5' is a hdf5 file" << std::endl; - CONDUIT_INFO("relay_io_example_hdf5_interface_1"); + END_EXAMPLE("relay_io_example_hdf5_interface_1"); } //----------------------------------------------------------------------------- -// 128-157 TEST(conduit_docs, relay_io_example_hdf5_interface_2) { - - CONDUIT_INFO("relay_io_example_hdf5_interface_opts"); - + BEGIN_EXAMPLE("relay_io_example_hdf5_interface_opts"); Node io_about; conduit::relay::io::about(io_about); std::cout << "\nRelay I/O Info and Default Options:" << std::endl; io_about.print(); - + Node &hdf5_opts = io_about["options/hdf5"]; // change the default chunking threshold to // a smaller number to enable compression for @@ -141,22 +137,20 @@ TEST(conduit_docs, relay_io_example_hdf5_interface_2) hdf5_opts.print(); // set options conduit::relay::io::hdf5_set_options(hdf5_opts); - + int num_vals = 5000; Node n; n["my_values"].set(DataType::float64(num_vals)); - + float64 *v_ptr = n["my_values"].value(); for(int i=0; i< num_vals; i++) { v_ptr[i] = float64(i); } - // save using options std::cout << "\nsaving data to 'myoutput_chunked.hdf5' " << std::endl; conduit::relay::io::hdf5_save(n,"myoutput_chunked.hdf5"); - - CONDUIT_INFO("relay_io_example_hdf5_interface_opts"); + END_EXAMPLE("relay_io_example_hdf5_interface_opts"); } diff --git a/src/tests/docs/t_conduit_docs_tutorial_basics.cpp b/src/tests/docs/t_conduit_docs_tutorial_basics.cpp index ab4e1546e..e7792e1af 100644 --- a/src/tests/docs/t_conduit_docs_tutorial_basics.cpp +++ b/src/tests/docs/t_conduit_docs_tutorial_basics.cpp @@ -51,45 +51,42 @@ #include "conduit.hpp" #include "conduit_blueprint.hpp" #include "conduit_relay.hpp" +#include "t_conduit_docs_tutorial_helpers.hpp" #include #include "gtest/gtest.h" #include "rapidjson/document.h" + using namespace conduit; //----------------------------------------------------------------------------- TEST(conduit_tutorial, basics_very_basic) { - CONDUIT_INFO("basics_very_basic"); - + BEGIN_EXAMPLE("basics_very_basic"); Node n; n["my"] = "data"; n.print(); - - CONDUIT_INFO("basics_very_basic"); + END_EXAMPLE("basics_very_basic"); } //----------------------------------------------------------------------------- TEST(conduit_tutorial, basics_hierarchial) { - CONDUIT_INFO("basics_hierarchial"); - + BEGIN_EXAMPLE("basics_hierarchial"); Node n; n["my"] = "data"; n["a/b/c"] = "d"; n["a"]["b"]["e"] = 64.0; n.print(); - - std::cout << "total bytes: " << n.total_strided_bytes() << std::endl; - CONDUIT_INFO("basics_hierarchial"); + std::cout << "total bytes: " << n.total_strided_bytes() << std::endl; + END_EXAMPLE("basics_hierarchial"); } //----------------------------------------------------------------------------- TEST(conduit_tutorial, basics_object_and_list) { - CONDUIT_INFO("basics_object_and_list"); - + BEGIN_EXAMPLE("basics_object_and_list"); Node n; n["object_example/val1"] = "data"; n["object_example/val2"] = 10u; @@ -102,15 +99,13 @@ TEST(conduit_tutorial, basics_object_and_list) } n.print(); - - CONDUIT_INFO("basics_object_and_list"); + END_EXAMPLE("basics_object_and_list"); } //----------------------------------------------------------------------------- TEST(conduit_tutorial, basics_object_and_list_itr) { - CONDUIT_INFO("basics_object_and_list_itr"); - + BEGIN_EXAMPLE("basics_object_and_list_itr"); Node n; n["object_example/val1"] = "data"; n["object_example/val2"] = 10u; @@ -121,34 +116,33 @@ TEST(conduit_tutorial, basics_object_and_list_itr) Node &list_entry = n["list_example"].append(); list_entry.set(i); } - + n.print(); - + NodeIterator itr = n["object_example"].children(); while(itr.has_next()) { Node &cld = itr.next(); std::string cld_name = itr.name(); - std::cout << cld_name << ": " << cld.to_json() << std::endl; + std::cout << cld_name << ": " << cld.to_string() << std::endl; } + std::cout << std::endl; + itr = n["list_example"].children(); while(itr.has_next()) { Node &cld = itr.next(); - std::cout << cld.to_json() << std::endl; + std::cout << cld.to_string() << std::endl; } - - - CONDUIT_INFO("basics_object_and_list_itr"); + END_EXAMPLE("basics_object_and_list_itr"); } //----------------------------------------------------------------------------- TEST(conduit_tutorial, basics_mem_spaces) { - CONDUIT_INFO("basics_mem_spaces"); - + BEGIN_EXAMPLE("basics_mem_spaces"); Node n; n["my"] = "data"; n["a/b/c"] = "d"; @@ -157,34 +151,29 @@ TEST(conduit_tutorial, basics_mem_spaces) Node ninfo; n.info(ninfo); ninfo.print(); - - CONDUIT_INFO("basics_mem_spaces"); + END_EXAMPLE("basics_mem_spaces"); } //----------------------------------------------------------------------------- TEST(conduit_tutorial, basics_bw_style) { - CONDUIT_INFO("basics_bw_style"); - + BEGIN_EXAMPLE("basics_bw_style"); Node n; uint32 val = 100; n["test"] = val; n.print(); n.print_detailed(); - - CONDUIT_INFO("basics_bw_style"); + END_EXAMPLE("basics_bw_style"); } //----------------------------------------------------------------------------- TEST(conduit_tutorial, basics_bw_style_from_native) { - CONDUIT_INFO("basics_bw_style_from_native"); - + BEGIN_EXAMPLE("basics_bw_style_from_native"); Node n; int val = 100; n["test"] = val; n.print_detailed(); - - CONDUIT_INFO("basics_bw_style_from_native"); + END_EXAMPLE("basics_bw_style_from_native"); } diff --git a/src/tests/docs/t_conduit_docs_tutorial_errors.cpp b/src/tests/docs/t_conduit_docs_tutorial_errors.cpp index 93e91c86f..23a340745 100644 --- a/src/tests/docs/t_conduit_docs_tutorial_errors.cpp +++ b/src/tests/docs/t_conduit_docs_tutorial_errors.cpp @@ -51,6 +51,7 @@ #include "conduit.hpp" #include "conduit_blueprint.hpp" #include "conduit_relay.hpp" +#include "t_conduit_docs_tutorial_helpers.hpp" #include #include "gtest/gtest.h" @@ -60,9 +61,11 @@ using namespace conduit; //----------------------------------------------------------------------------- TEST(conduit_tutorial, error_handlers_funcs_start) { - CONDUIT_INFO("error_handlers_funcs"); + BEGIN_EXAMPLE("error_handlers_funcs"); } +// _conduit_error_handlers_funcs_start + //----------------------------------------------------------------------------- void my_info_handler(const std::string &msg, const std::string &file, @@ -88,26 +91,27 @@ void my_error_handler(const std::string &msg, throw conduit::Error(msg,file,line); } +// _conduit_error_handlers_funcs_end + //----------------------------------------------------------------------------- TEST(conduit_tutorial, error_handlers_funcs_end) { - CONDUIT_INFO("error_handlers_funcs"); + END_EXAMPLE("error_handlers_funcs"); } //----------------------------------------------------------------------------- TEST(conduit_tutorial, error_handlers) { - CONDUIT_INFO("error_handlers"); - + BEGIN_EXAMPLE("error_handlers_rewire"); // rewire error handlers conduit::utils::set_info_handler(my_info_handler); conduit::utils::set_warning_handler(my_warning_handler); conduit::utils::set_error_handler(my_error_handler); - + // emit an example info message CONDUIT_INFO("An info message"); - + Node n; n["my_value"].set_float64(42.0); @@ -128,13 +132,14 @@ TEST(conduit_tutorial, error_handlers) { // pass } - + END_EXAMPLE("error_handlers_rewire"); + + BEGIN_EXAMPLE("error_handlers_reset"); // restore default handlers conduit::utils::set_info_handler(conduit::utils::default_info_handler); conduit::utils::set_warning_handler(conduit::utils::default_warning_handler); conduit::utils::set_error_handler(conduit::utils::default_error_handler); - - CONDUIT_INFO("error_handlers"); + END_EXAMPLE("error_handlers_reset"); } diff --git a/src/tests/docs/t_conduit_docs_tutorial_numeric.cpp b/src/tests/docs/t_conduit_docs_tutorial_numeric.cpp index cf218c6fb..254273d43 100644 --- a/src/tests/docs/t_conduit_docs_tutorial_numeric.cpp +++ b/src/tests/docs/t_conduit_docs_tutorial_numeric.cpp @@ -51,6 +51,7 @@ #include "conduit.hpp" #include "conduit_blueprint.hpp" #include "conduit_relay.hpp" +#include "t_conduit_docs_tutorial_helpers.hpp" #include #include "gtest/gtest.h" @@ -60,41 +61,30 @@ using namespace conduit; //----------------------------------------------------------------------------- TEST(conduit_tutorial, numeric_as_dtype) { - CONDUIT_INFO("numeric_as_dtype"); - - // _conduit_tutorial_cpp_numeric_node_as_dtype_start + BEGIN_EXAMPLE("numeric_as_dtype"); Node n; int64 val = 100; n = val; std::cout << n.as_int64() << std::endl; - // _conduit_tutorial_cpp_numeric_node_as_dtype_end - - CONDUIT_INFO("numeric_as_dtype"); + END_EXAMPLE("numeric_as_dtype"); } //----------------------------------------------------------------------------- TEST(conduit_tutorial, numeric_via_value) { - CONDUIT_INFO("numeric_via_value"); - - // _conduit_tutorial_cpp_numeric_node_value_start + BEGIN_EXAMPLE("numeric_via_value"); Node n; int64 val = 100; n = val; int64 my_val = n.value(); std::cout << my_val << std::endl; - // _conduit_tutorial_cpp_numeric_node_value_end - - CONDUIT_INFO("numeric_via_value"); + END_EXAMPLE("numeric_via_value"); } - //----------------------------------------------------------------------------- TEST(conduit_tutorial, numeric_ptr_as_dtype) { - CONDUIT_INFO("numeric_ptr_as_dtype"); - - // _conduit_tutorial_cpp_numeric_array_node_value_start + BEGIN_EXAMPLE("numeric_ptr_as_dtype"); int64 vals[4] = {100,200,300,400}; Node n; @@ -106,17 +96,13 @@ TEST(conduit_tutorial, numeric_ptr_as_dtype) { std::cout << "my_vals[" << i << "] = " << my_vals[i] << std::endl; } - // _conduit_tutorial_cpp_numeric_array_node_value_end - - CONDUIT_INFO("numeric_ptr_as_dtype"); + END_EXAMPLE("numeric_ptr_as_dtype"); } //----------------------------------------------------------------------------- TEST(conduit_tutorial, numeric_ptr_via_value) { - CONDUIT_INFO("numeric_ptr_via_value"); - - // _conduit_tutorial_cpp_numeric_array_node_value_start + BEGIN_EXAMPLE("numeric_ptr_via_value"); int64 vals[4] = {100,200,300,400}; Node n; @@ -128,41 +114,35 @@ TEST(conduit_tutorial, numeric_ptr_via_value) { std::cout << "my_vals[" << i << "] = " << my_vals[i] << std::endl; } - // _conduit_tutorial_cpp_numeric_array_node_value_end - - CONDUIT_INFO("numeric_ptr_via_value"); + END_EXAMPLE("numeric_ptr_via_value"); } //----------------------------------------------------------------------------- TEST(conduit_tutorial, numeric_strided_data_array) { - CONDUIT_INFO("numeric_strided_data_array"); - - // _conduit_tutorial_cpp_numeric_non-contiguous_start + BEGIN_EXAMPLE("numeric_strided_data_array"); int64 vals[4] = {100,200,300,400}; Node n; n.set(vals,2, // # of elements 0, // offset in bytes sizeof(int64)*2); // stride in bytes - + int64_array my_vals = n.value(); for(index_t i=0; i < 2; i++) { std::cout << "my_vals[" << i << "] = " << my_vals[i] << std::endl; } - - my_vals.print(); - // _conduit_tutorial_cpp_numeric_non-contiguous_end - CONDUIT_INFO("numeric_strided_data_array"); + my_vals.print(); + END_EXAMPLE("numeric_strided_data_array"); } //----------------------------------------------------------------------------- TEST(conduit_tutorial, numeric_double_conversion_start) { - CONDUIT_INFO("numeric_double_conversion_start"); + BEGIN_EXAMPLE("numeric_double_conversion"); } // _conduit_tutorial_cpp_numeric_introspection_start @@ -230,6 +210,6 @@ TEST(conduit_tutorial, numeric_double_conversion) //----------------------------------------------------------------------------- TEST(conduit_tutorial, numeric_double_conversion_end) { - CONDUIT_INFO("numeric_double_conversion_end"); + END_EXAMPLE("numeric_double_conversion"); } diff --git a/src/tests/docs/t_conduit_docs_tutorial_ownership.cpp b/src/tests/docs/t_conduit_docs_tutorial_ownership.cpp index 67f25bd21..5ff3c6920 100644 --- a/src/tests/docs/t_conduit_docs_tutorial_ownership.cpp +++ b/src/tests/docs/t_conduit_docs_tutorial_ownership.cpp @@ -51,6 +51,7 @@ #include "conduit.hpp" #include "conduit_blueprint.hpp" #include "conduit_relay.hpp" +#include "t_conduit_docs_tutorial_helpers.hpp" #include #include "gtest/gtest.h" @@ -60,8 +61,7 @@ using namespace conduit; //----------------------------------------------------------------------------- TEST(conduit_tutorial, mem_ownership_external) { - CONDUIT_INFO("mem_ownership_external"); - + BEGIN_EXAMPLE("mem_ownership_external"); int vsize = 5; std::vector vals(vsize,0.0); for(int i=0;i Date: Fri, 24 Apr 2020 21:37:28 -0700 Subject: [PATCH 5/5] add support to write and read lists from hdf5 (#554) * working towards list support for hdf5 * working list support * docs updates and more testing * add more hdf5 list tests --- CHANGELOG.md | 1 + .../t_conduit_docs_blueprint_demos_out.txt | 16 +- .../t_conduit_docs_blueprint_examples_out.txt | 12 +- ...uit_docs_relay_io_generic_examples_out.txt | 8 +- ...duit_docs_relay_io_handle_examples_out.txt | 6 +- ...onduit_docs_relay_io_hdf5_examples_out.txt | 7 +- .../t_conduit_docs_tutorial_basics_out.txt | 10 +- .../t_conduit_docs_tutorial_errors_out.txt | 6 +- .../t_conduit_docs_tutorial_numeric_out.txt | 6 +- .../t_conduit_docs_tutorial_ownership_out.txt | 4 +- .../t_conduit_docs_tutorial_parse_out.txt | 20 +- ...onduit_docs_tutorial_python_basics_out.txt | 12 +- ...nduit_docs_tutorial_python_numeric_out.txt | 6 +- ...uit_docs_tutorial_python_ownership_out.txt | 10 +- ...conduit_docs_tutorial_python_parse_out.txt | 6 +- ...al_python_relay_io_handle_examples_out.txt | 4 +- ...tutorial_python_relay_mpi_examples_out.txt | 12 +- src/libs/relay/conduit_relay_io_hdf5.cpp | 575 ++++++++++++++---- .../t_conduit_docs_relay_io_hdf5_examples.cpp | 13 +- src/tests/relay/t_relay_io_hdf5.cpp | 433 +++++++------ 20 files changed, 767 insertions(+), 400 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fdd725f47..7c049a6d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ and this project aspires to adhere to [Semantic Versioning](https://semver.org/s #### Relay - Added an open mode option to RelayIOHandle. See RelayIOHandle docs (https://llnl-conduit.readthedocs.io/en/latest/relay_io.html#relay-i-o-handle-interface) for more details. - Added the conduit.relay.mpi Python module to support Relay MPI in Python. +- Added support to write and read Conduit lists to HDF5 files. Since HDF5 Groups do not support unnamed indexed children, each list child is written using a string name that represents its index and a special attribute is written to the HDF5 group to mark the list case. On read, the special attribute is used to detect and read this style of group back into a Conduit list. #### Blueprint - Added support for Adjacency sets for Structured Mesh Topologies. See the `blueprint::mesh::examples::adjset_uniform` example. diff --git a/src/docs/sphinx/t_conduit_docs_blueprint_demos_out.txt b/src/docs/sphinx/t_conduit_docs_blueprint_demos_out.txt index 19a65aea8..50e83affc 100644 --- a/src/docs/sphinx/t_conduit_docs_blueprint_demos_out.txt +++ b/src/docs/sphinx/t_conduit_docs_blueprint_demos_out.txt @@ -31,7 +31,7 @@ fields: END_EXAMPLE("blueprint_demo_basic_uniform") [/Users/harrison37/Work/github/llnl/conduit/src/tests/docs/t_conduit_docs_blueprint_demos.cpp : 107] Testing Basic Example 'uniform' -[ OK ] conduit_docs.blueprint_demo_basic_uniform (8 ms) +[ OK ] conduit_docs.blueprint_demo_basic_uniform (9 ms) [ RUN ] conduit_docs.blueprint_demo_basic_rectilinear BEGIN_EXAMPLE("blueprint_demo_basic_rectilinear") @@ -55,7 +55,7 @@ fields: END_EXAMPLE("blueprint_demo_basic_rectilinear") [/Users/harrison37/Work/github/llnl/conduit/src/tests/docs/t_conduit_docs_blueprint_demos.cpp : 107] Testing Basic Example 'rectilinear' -[ OK ] conduit_docs.blueprint_demo_basic_rectilinear (5 ms) +[ OK ] conduit_docs.blueprint_demo_basic_rectilinear (6 ms) [ RUN ] conduit_docs.blueprint_demo_basic_structured BEGIN_EXAMPLE("blueprint_demo_basic_structured") @@ -110,7 +110,7 @@ fields: END_EXAMPLE("blueprint_demo_basic_tris") [/Users/harrison37/Work/github/llnl/conduit/src/tests/docs/t_conduit_docs_blueprint_demos.cpp : 107] Testing Basic Example 'tris' -[ OK ] conduit_docs.blueprint_demo_basic_tris (6 ms) +[ OK ] conduit_docs.blueprint_demo_basic_tris (7 ms) [ RUN ] conduit_docs.blueprint_demo_basic_quads BEGIN_EXAMPLE("blueprint_demo_basic_quads") @@ -193,7 +193,7 @@ fields: END_EXAMPLE("blueprint_demo_basic_hexs") [/Users/harrison37/Work/github/llnl/conduit/src/tests/docs/t_conduit_docs_blueprint_demos.cpp : 107] Testing Basic Example 'hexs' -[ OK ] conduit_docs.blueprint_demo_basic_hexs (6 ms) +[ OK ] conduit_docs.blueprint_demo_basic_hexs (7 ms) [ RUN ] conduit_docs.blueprint_demo_basic_polygons BEGIN_EXAMPLE("blueprint_demo_basic_polygons") @@ -220,7 +220,7 @@ fields: END_EXAMPLE("blueprint_demo_basic_polygons") [/Users/harrison37/Work/github/llnl/conduit/src/tests/docs/t_conduit_docs_blueprint_demos.cpp : 107] Testing Basic Example 'polygons' -[ OK ] conduit_docs.blueprint_demo_basic_polygons (6 ms) +[ OK ] conduit_docs.blueprint_demo_basic_polygons (7 ms) [ RUN ] conduit_docs.blueprint_demo_basic_polyhedra BEGIN_EXAMPLE("blueprint_demo_basic_polyhedra") @@ -279,9 +279,9 @@ fields: values: [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0] END_EXAMPLE("blueprint_demo_basic_uniform_detailed") -[ OK ] conduit_docs.blueprint_demo_basic_uniform_detailed (2 ms) -[----------] 10 tests from conduit_docs (62 ms total) +[ OK ] conduit_docs.blueprint_demo_basic_uniform_detailed (3 ms) +[----------] 10 tests from conduit_docs (68 ms total) [----------] Global test environment tear-down -[==========] 10 tests from 1 test suite ran. (62 ms total) +[==========] 10 tests from 1 test suite ran. (68 ms total) [ PASSED ] 10 tests. diff --git a/src/docs/sphinx/t_conduit_docs_blueprint_examples_out.txt b/src/docs/sphinx/t_conduit_docs_blueprint_examples_out.txt index 944789b81..b43d4f242 100644 --- a/src/docs/sphinx/t_conduit_docs_blueprint_examples_out.txt +++ b/src/docs/sphinx/t_conduit_docs_blueprint_examples_out.txt @@ -18,7 +18,7 @@ coords: valid: "true" END_EXAMPLE("blueprint_example_1") -[ OK ] conduit_docs.blueprint_example_1 (1 ms) +[ OK ] conduit_docs.blueprint_example_1 (2 ms) [ RUN ] conduit_docs.blueprint_example_2 BEGIN_EXAMPLE("blueprint_example_2") example 'separate' mcarray @@ -29,15 +29,15 @@ z: [3.0, 3.0, 3.0, 3.0, 3.0] mem_spaces: - 0x7fc12cc04e20: + 0x7fcf69404e20: path: "x" type: "allocated" bytes: 40 - 0x7fc12cc04f20: + 0x7fcf69404f20: path: "y" type: "allocated" bytes: 40 - 0x7fc12cc05080: + 0x7fcf69405080: path: "z" type: "allocated" bytes: 40 @@ -55,7 +55,7 @@ transformed to 'interleaved' mcarray } mem_spaces: - 0x7fc12cc06b70: + 0x7fcf69406b70: path: "" type: "allocated" bytes: 120 @@ -65,7 +65,7 @@ total_bytes_compact: 120 total_strided_bytes: 312 END_EXAMPLE("blueprint_example_2") -[ OK ] conduit_docs.blueprint_example_2 (1 ms) +[ OK ] conduit_docs.blueprint_example_2 (0 ms) [----------] 2 tests from conduit_docs (2 ms total) [----------] Global test environment tear-down diff --git a/src/docs/sphinx/t_conduit_docs_relay_io_generic_examples_out.txt b/src/docs/sphinx/t_conduit_docs_relay_io_generic_examples_out.txt index 2933816f5..2377f5565 100644 --- a/src/docs/sphinx/t_conduit_docs_relay_io_generic_examples_out.txt +++ b/src/docs/sphinx/t_conduit_docs_relay_io_generic_examples_out.txt @@ -69,7 +69,7 @@ a: new_data: 42.0 END_EXAMPLE("relay_io_example_2_hdf5") -[ OK ] conduit_docs.relay_io_example_2_hdf5 (3 ms) +[ OK ] conduit_docs.relay_io_example_2_hdf5 (2 ms) [ RUN ] conduit_docs.relay_io_example_3_hdf5 BEGIN_EXAMPLE("relay_io_example_3_hdf5") @@ -97,7 +97,7 @@ a: my_data: 1.0 END_EXAMPLE("relay_io_example_3_hdf5") -[ OK ] conduit_docs.relay_io_example_3_hdf5 (1 ms) +[ OK ] conduit_docs.relay_io_example_3_hdf5 (2 ms) [ RUN ] conduit_docs.relay_io_example_4_hdf5 BEGIN_EXAMPLE("relay_io_example_4_hdf5") @@ -113,7 +113,7 @@ Load result from 'path/to' my_data: 1.0 END_EXAMPLE("relay_io_example_4_hdf5") -[ OK ] conduit_docs.relay_io_example_4_hdf5 (2 ms) +[ OK ] conduit_docs.relay_io_example_4_hdf5 (1 ms) [ RUN ] conduit_docs.relay_io_example_5_hdf5 BEGIN_EXAMPLE("relay_io_example_5_hdf5") @@ -129,7 +129,7 @@ path: my_data: 1.0 END_EXAMPLE("relay_io_example_5_hdf5") -[ OK ] conduit_docs.relay_io_example_5_hdf5 (1 ms) +[ OK ] conduit_docs.relay_io_example_5_hdf5 (2 ms) [----------] 6 tests from conduit_docs (11 ms total) [----------] Global test environment tear-down diff --git a/src/docs/sphinx/t_conduit_docs_relay_io_handle_examples_out.txt b/src/docs/sphinx/t_conduit_docs_relay_io_handle_examples_out.txt index c9b0bb57f..1bbbc96ce 100644 --- a/src/docs/sphinx/t_conduit_docs_relay_io_handle_examples_out.txt +++ b/src/docs/sphinx/t_conduit_docs_relay_io_handle_examples_out.txt @@ -33,9 +33,9 @@ a: c: 42.0 END_EXAMPLE("relay_io_handle_example_1") -[ OK ] conduit_docs.relay_io_handle_1 (5 ms) -[----------] 1 test from conduit_docs (5 ms total) +[ OK ] conduit_docs.relay_io_handle_1 (6 ms) +[----------] 1 test from conduit_docs (6 ms total) [----------] Global test environment tear-down -[==========] 1 test from 1 test suite ran. (5 ms total) +[==========] 1 test from 1 test suite ran. (7 ms total) [ PASSED ] 1 test. diff --git a/src/docs/sphinx/t_conduit_docs_relay_io_hdf5_examples_out.txt b/src/docs/sphinx/t_conduit_docs_relay_io_hdf5_examples_out.txt index cefd7b344..b07f56cb8 100644 --- a/src/docs/sphinx/t_conduit_docs_relay_io_hdf5_examples_out.txt +++ b/src/docs/sphinx/t_conduit_docs_relay_io_hdf5_examples_out.txt @@ -18,6 +18,8 @@ Path 'myoutput.hdf5:a/my_data' exists Data loaded: 1.0 +Path 'myoutput.hdf5:a/my_list/0' exists + Data loaded: a: @@ -25,6 +27,9 @@ a: b: my_string: "value" c: 144 + my_list: + - 42.0 + - 42 File 'myoutput.hdf5' is a hdf5 file @@ -75,7 +80,7 @@ chunking: saving data to 'myoutput_chunked.hdf5' END_EXAMPLE("relay_io_example_hdf5_interface_opts") -[ OK ] conduit_docs.relay_io_example_hdf5_interface_2 (3 ms) +[ OK ] conduit_docs.relay_io_example_hdf5_interface_2 (2 ms) [----------] 2 tests from conduit_docs (7 ms total) [----------] Global test environment tear-down diff --git a/src/docs/sphinx/t_conduit_docs_tutorial_basics_out.txt b/src/docs/sphinx/t_conduit_docs_tutorial_basics_out.txt index b8195da09..85d6b297f 100644 --- a/src/docs/sphinx/t_conduit_docs_tutorial_basics_out.txt +++ b/src/docs/sphinx/t_conduit_docs_tutorial_basics_out.txt @@ -36,7 +36,7 @@ list_example: - 4 END_EXAMPLE("basics_object_and_list") -[ OK ] conduit_tutorial.basics_object_and_list (0 ms) +[ OK ] conduit_tutorial.basics_object_and_list (1 ms) [ RUN ] conduit_tutorial.basics_object_and_list_itr BEGIN_EXAMPLE("basics_object_and_list_itr") @@ -66,15 +66,15 @@ END_EXAMPLE("basics_object_and_list_itr") BEGIN_EXAMPLE("basics_mem_spaces") mem_spaces: - 0x7f9239c04e40: + 0x7fa457501230: path: "my" type: "allocated" bytes: 5 - 0x7f9239c05420: + 0x7fa457501810: path: "a/b/c" type: "allocated" bytes: 2 - 0x7f9239c05e50: + 0x7fa457502240: path: "a/b/e" type: "allocated" bytes: 8 @@ -104,7 +104,7 @@ BEGIN_EXAMPLE("basics_bw_style_from_native") } END_EXAMPLE("basics_bw_style_from_native") [ OK ] conduit_tutorial.basics_bw_style_from_native (0 ms) -[----------] 7 tests from conduit_tutorial (0 ms total) +[----------] 7 tests from conduit_tutorial (1 ms total) [----------] Global test environment tear-down [==========] 7 tests from 1 test suite ran. (1 ms total) diff --git a/src/docs/sphinx/t_conduit_docs_tutorial_errors_out.txt b/src/docs/sphinx/t_conduit_docs_tutorial_errors_out.txt index 4b5b491f0..edd46640a 100644 --- a/src/docs/sphinx/t_conduit_docs_tutorial_errors_out.txt +++ b/src/docs/sphinx/t_conduit_docs_tutorial_errors_out.txt @@ -16,9 +16,9 @@ BEGIN_EXAMPLE("error_handlers_rewire") END_EXAMPLE("error_handlers_rewire") BEGIN_EXAMPLE("error_handlers_reset") END_EXAMPLE("error_handlers_reset") -[ OK ] conduit_tutorial.error_handlers (0 ms) -[----------] 3 tests from conduit_tutorial (0 ms total) +[ OK ] conduit_tutorial.error_handlers (1 ms) +[----------] 3 tests from conduit_tutorial (1 ms total) [----------] Global test environment tear-down -[==========] 3 tests from 1 test suite ran. (0 ms total) +[==========] 3 tests from 1 test suite ran. (1 ms total) [ PASSED ] 3 tests. diff --git a/src/docs/sphinx/t_conduit_docs_tutorial_numeric_out.txt b/src/docs/sphinx/t_conduit_docs_tutorial_numeric_out.txt index 42be0963e..0d11c6363 100644 --- a/src/docs/sphinx/t_conduit_docs_tutorial_numeric_out.txt +++ b/src/docs/sphinx/t_conduit_docs_tutorial_numeric_out.txt @@ -53,12 +53,12 @@ vals[0] = 1000 vals[1] = 2000 vals[2] = 3000 vals[3] = 4000 -[ OK ] conduit_tutorial.numeric_double_conversion (1 ms) +[ OK ] conduit_tutorial.numeric_double_conversion (0 ms) [ RUN ] conduit_tutorial.numeric_double_conversion_end END_EXAMPLE("numeric_double_conversion") [ OK ] conduit_tutorial.numeric_double_conversion_end (0 ms) -[----------] 8 tests from conduit_tutorial (1 ms total) +[----------] 8 tests from conduit_tutorial (0 ms total) [----------] Global test environment tear-down -[==========] 8 tests from 1 test suite ran. (1 ms total) +[==========] 8 tests from 1 test suite ran. (0 ms total) [ PASSED ] 8 tests. diff --git a/src/docs/sphinx/t_conduit_docs_tutorial_ownership_out.txt b/src/docs/sphinx/t_conduit_docs_tutorial_ownership_out.txt index aaad41d38..7217ac0fa 100644 --- a/src/docs/sphinx/t_conduit_docs_tutorial_ownership_out.txt +++ b/src/docs/sphinx/t_conduit_docs_tutorial_ownership_out.txt @@ -6,11 +6,11 @@ Running main() from /Users/harrison37/Work/github/llnl/conduit/src/blt/thirdpart BEGIN_EXAMPLE("mem_ownership_external") mem_spaces: - 0x7fd547c04c50: + 0x7f9fe6c04c50: path: "v_owned" type: "allocated" bytes: 40 - 0x7fd547c04890: + 0x7f9fe6c04890: path: "v_external" type: "external" total_bytes_allocated: 40 diff --git a/src/docs/sphinx/t_conduit_docs_tutorial_parse_out.txt b/src/docs/sphinx/t_conduit_docs_tutorial_parse_out.txt index 4f7219c34..33b8d35eb 100644 --- a/src/docs/sphinx/t_conduit_docs_tutorial_parse_out.txt +++ b/src/docs/sphinx/t_conduit_docs_tutorial_parse_out.txt @@ -19,7 +19,7 @@ BEGIN_EXAMPLE("t_conduit_docs_tutorial_json") "mykey": {"dtype":"float64", "number_of_elements": 1, "offset": 0, "stride": 8, "element_bytes": 8, "endianness": "little", "value": 42.0} } END_EXAMPLE("t_conduit_docs_tutorial_json") -[ OK ] conduit_tutorial.t_conduit_docs_tutorial_json (0 ms) +[ OK ] conduit_tutorial.t_conduit_docs_tutorial_json (1 ms) [ RUN ] conduit_tutorial.t_conduit_docs_tutorial_yaml_inline_array BEGIN_EXAMPLE("t_conduit_docs_tutorial_yaml_inline_array") [0.0, 10.0, 20.0, 30.0] @@ -75,7 +75,7 @@ b: 200.0 mem_spaces: - 0x7ffeea50a030: + 0x7ffee952f030: path: "a" type: "external" total_bytes_allocated: 0 @@ -84,14 +84,14 @@ total_bytes_compact: 16 total_strided_bytes: 16 END_EXAMPLE("t_json_generator_bind_to_incore") -[ OK ] conduit_tutorial.json_generator_bind_to_incore (1 ms) +[ OK ] conduit_tutorial.json_generator_bind_to_incore (0 ms) [ RUN ] conduit_tutorial.json_generator_compact BEGIN_EXAMPLE("t_json_generator_compact") [100.0, 200.0, 300.0, 400.0, 500.0] [-100.0, -200.0, -300.0, -400.0, -500.0] mem_spaces: - 0x7ffeea509ff0: + 0x7ffee952eff0: path: "" type: "external" total_bytes_allocated: 0 @@ -101,7 +101,7 @@ total_strided_bytes: 72 mem_spaces: - 0x7ffeea509ff0: + 0x7ffee952eff0: path: "" type: "external" total_bytes_allocated: 0 @@ -113,7 +113,7 @@ total_strided_bytes: 72 {"dtype":"float64", "number_of_elements": 5, "offset": 0, "stride": 8, "element_bytes": 8, "endianness": "little"} mem_spaces: - 0x7fa1db4056e0: + 0x7feb244056e0: path: "" type: "allocated" bytes: 40 @@ -125,7 +125,7 @@ total_strided_bytes: 40 [-100.0, -200.0, -300.0, -400.0, -500.0] mem_spaces: - 0x7fa1db406350: + 0x7feb24406350: path: "" type: "allocated" bytes: 40 @@ -135,9 +135,9 @@ total_bytes_compact: 40 total_strided_bytes: 40 END_EXAMPLE("t_json_generator_compact") -[ OK ] conduit_tutorial.json_generator_compact (0 ms) -[----------] 8 tests from conduit_tutorial (1 ms total) +[ OK ] conduit_tutorial.json_generator_compact (1 ms) +[----------] 8 tests from conduit_tutorial (2 ms total) [----------] Global test environment tear-down -[==========] 8 tests from 1 test suite ran. (1 ms total) +[==========] 8 tests from 1 test suite ran. (2 ms total) [ PASSED ] 8 tests. diff --git a/src/docs/sphinx/t_conduit_docs_tutorial_python_basics_out.txt b/src/docs/sphinx/t_conduit_docs_tutorial_python_basics_out.txt index 3d3574669..0e2535f36 100644 --- a/src/docs/sphinx/t_conduit_docs_tutorial_python_basics_out.txt +++ b/src/docs/sphinx/t_conduit_docs_tutorial_python_basics_out.txt @@ -25,7 +25,7 @@ test 106 106: test_007_basics_bw_style_from_native (t_conduit_docs_tutorial_python_basics.Conduit_Tutorial_Python_Basics) ... ok 106: 106: ---------------------------------------------------------------------- -106: Ran 7 tests in 0.001s +106: Ran 7 tests in 0.003s 106: 106: OK 106: BEGIN_EXAMPLE("py_basics_very_basic") @@ -84,15 +84,15 @@ test 106 106: BEGIN_EXAMPLE("py_basics_mem_spaces") 106: 106: mem_spaces: -106: 0x7ff41ae58e90: +106: 0x7fc64573faa0: 106: path: "my" 106: type: "allocated" 106: bytes: 5 -106: 0x7ff41ae84120: +106: 0x7fc645745590: 106: path: "a/b/c" 106: type: "allocated" 106: bytes: 2 -106: 0x7ff41ae7d8a0: +106: 0x7fc645741e50: 106: path: "a/b/e" 106: type: "allocated" 106: bytes: 8 @@ -113,11 +113,11 @@ test 106 106: "test": {"dtype":"int64", "number_of_elements": 1, "offset": 0, "stride": 8, "element_bytes": 8, "endianness": "little"} 106: } 106: END_EXAMPLE("py_basics_bw_style_from_native") -1/1 Test #106: t_conduit_docs_tutorial_python_basics ... Passed 0.15 sec +1/1 Test #106: t_conduit_docs_tutorial_python_basics ... Passed 0.34 sec The following tests passed: t_conduit_docs_tutorial_python_basics 100% tests passed, 0 tests failed out of 1 -Total Test time (real) = 0.17 sec +Total Test time (real) = 0.40 sec diff --git a/src/docs/sphinx/t_conduit_docs_tutorial_python_numeric_out.txt b/src/docs/sphinx/t_conduit_docs_tutorial_python_numeric_out.txt index e2ead3087..f12eb4ec2 100644 --- a/src/docs/sphinx/t_conduit_docs_tutorial_python_numeric_out.txt +++ b/src/docs/sphinx/t_conduit_docs_tutorial_python_numeric_out.txt @@ -19,17 +19,17 @@ test 107 107: test_001_numeric_via_value (t_conduit_docs_tutorial_python_numeric.Conduit_Tutorial_Python_Numeric) ... ok 107: 107: ---------------------------------------------------------------------- -107: Ran 1 test in 0.000s +107: Ran 1 test in 0.001s 107: 107: OK 107: BEGIN_EXAMPLE("py_numeric_via_value") 107: 10 107: END_EXAMPLE("py_numeric_via_value") -1/1 Test #107: t_conduit_docs_tutorial_python_numeric ... Passed 0.14 sec +1/1 Test #107: t_conduit_docs_tutorial_python_numeric ... Passed 0.16 sec The following tests passed: t_conduit_docs_tutorial_python_numeric 100% tests passed, 0 tests failed out of 1 -Total Test time (real) = 0.17 sec +Total Test time (real) = 0.18 sec diff --git a/src/docs/sphinx/t_conduit_docs_tutorial_python_ownership_out.txt b/src/docs/sphinx/t_conduit_docs_tutorial_python_ownership_out.txt index 59dee2579..b80c5bdd5 100644 --- a/src/docs/sphinx/t_conduit_docs_tutorial_python_ownership_out.txt +++ b/src/docs/sphinx/t_conduit_docs_tutorial_python_ownership_out.txt @@ -19,17 +19,17 @@ test 109 109: test_001_mem_ownership_external (t_conduit_docs_tutorial_python_ownership.Conduit_Tutorial_Python_Ownership) ... ok 109: 109: ---------------------------------------------------------------------- -109: Ran 1 test in 0.001s +109: Ran 1 test in 0.002s 109: 109: OK 109: BEGIN_EXAMPLE("py_mem_ownership_external") 109: 109: mem_spaces: -109: 0x7fe138e8b760: +109: 0x7f92f621ad90: 109: path: "v_owned" 109: type: "allocated" 109: bytes: 40 -109: 0x7fe138e8d3f0: +109: 0x7f92f62484f0: 109: path: "v_external" 109: type: "external" 109: total_bytes_allocated: 40 @@ -47,11 +47,11 @@ test 109 109: 109: [3.1415 0. 0. 0. 0. ] 109: END_EXAMPLE("py_mem_ownership_external") -1/1 Test #109: t_conduit_docs_tutorial_python_ownership ... Passed 0.15 sec +1/1 Test #109: t_conduit_docs_tutorial_python_ownership ... Passed 0.19 sec The following tests passed: t_conduit_docs_tutorial_python_ownership 100% tests passed, 0 tests failed out of 1 -Total Test time (real) = 0.17 sec +Total Test time (real) = 0.21 sec diff --git a/src/docs/sphinx/t_conduit_docs_tutorial_python_parse_out.txt b/src/docs/sphinx/t_conduit_docs_tutorial_python_parse_out.txt index e55cef54e..73e909d6f 100644 --- a/src/docs/sphinx/t_conduit_docs_tutorial_python_parse_out.txt +++ b/src/docs/sphinx/t_conduit_docs_tutorial_python_parse_out.txt @@ -24,7 +24,7 @@ test 108 108: test_json_generator_std (t_conduit_docs_tutorial_python_parse.Conduit_Tutorial_Python_Parse) ... ok 108: 108: ---------------------------------------------------------------------- -108: Ran 6 tests in 0.001s +108: Ran 6 tests in 0.002s 108: 108: OK 108: BEGIN_EXAMPLE("py_conduit_docs_tutorial_json") @@ -63,11 +63,11 @@ test 108 108: test: 100.0 108: 108: END_EXAMPLE("py_json_generator_std") -1/1 Test #108: t_conduit_docs_tutorial_python_parse ... Passed 0.14 sec +1/1 Test #108: t_conduit_docs_tutorial_python_parse ... Passed 0.17 sec The following tests passed: t_conduit_docs_tutorial_python_parse 100% tests passed, 0 tests failed out of 1 -Total Test time (real) = 0.17 sec +Total Test time (real) = 0.20 sec diff --git a/src/docs/sphinx/t_conduit_docs_tutorial_python_relay_io_handle_examples_out.txt b/src/docs/sphinx/t_conduit_docs_tutorial_python_relay_io_handle_examples_out.txt index e9ec1b3f0..73f99fe71 100644 --- a/src/docs/sphinx/t_conduit_docs_tutorial_python_relay_io_handle_examples_out.txt +++ b/src/docs/sphinx/t_conduit_docs_tutorial_python_relay_io_handle_examples_out.txt @@ -54,11 +54,11 @@ test 110 110: 110: 110: END_EXAMPLE("py_relay_io_handle") -1/1 Test #110: t_conduit_docs_tutorial_python_relay_io_handle_examples ... Passed 0.21 sec +1/1 Test #110: t_conduit_docs_tutorial_python_relay_io_handle_examples ... Passed 0.30 sec The following tests passed: t_conduit_docs_tutorial_python_relay_io_handle_examples 100% tests passed, 0 tests failed out of 1 -Total Test time (real) = 0.23 sec +Total Test time (real) = 0.32 sec diff --git a/src/docs/sphinx/t_conduit_docs_tutorial_python_relay_mpi_examples_out.txt b/src/docs/sphinx/t_conduit_docs_tutorial_python_relay_mpi_examples_out.txt index a1ab52c5c..52a1fc1f2 100644 --- a/src/docs/sphinx/t_conduit_docs_tutorial_python_relay_mpi_examples_out.txt +++ b/src/docs/sphinx/t_conduit_docs_tutorial_python_relay_mpi_examples_out.txt @@ -54,8 +54,6 @@ test 111 111: b: 111: my_string: "value" 111: -111: -111: END_EXAMPLE("py_mpi_bcast_using_schema") 111: [rank: 1] received: 111: a: 111: data: 1.0 @@ -63,6 +61,8 @@ test 111 111: b: 111: my_string: "value" 111: +111: +111: END_EXAMPLE("py_mpi_bcast_using_schema") 111: ok 111: test_004_mpi_bcast_using_schema (t_conduit_docs_tutorial_python_relay_mpi_examples.Conduit_Tutorial_Python_Relay_IO_Handle) ... ok 111: test_004_mpi_bcast_using_schema (t_conduit_docs_tutorial_python_relay_mpi_examples.Conduit_Tutorial_Python_Relay_IO_Handle) ... @@ -91,20 +91,20 @@ test 111 111: ok 111: 111: ---------------------------------------------------------------------- -111: Ran 6 tests in 0.083s +111: Ran 6 tests in 0.126s 111: 111: OK 111: ok 111: 111: ---------------------------------------------------------------------- -111: Ran 6 tests in 0.084s +111: Ran 6 tests in 0.135s 111: 111: OK -1/1 Test #111: t_conduit_docs_tutorial_python_relay_mpi_examples ... Passed 0.26 sec +1/1 Test #111: t_conduit_docs_tutorial_python_relay_mpi_examples ... Passed 0.39 sec The following tests passed: t_conduit_docs_tutorial_python_relay_mpi_examples 100% tests passed, 0 tests failed out of 1 -Total Test time (real) = 0.28 sec +Total Test time (real) = 0.42 sec diff --git a/src/libs/relay/conduit_relay_io_hdf5.cpp b/src/libs/relay/conduit_relay_io_hdf5.cpp index 52a06edef..c1238dc1d 100644 --- a/src/libs/relay/conduit_relay_io_hdf5.cpp +++ b/src/libs/relay/conduit_relay_io_hdf5.cpp @@ -161,6 +161,10 @@ namespace mpi namespace io { + +static std::string conduit_hdf5_list_attr_name = "__conduit_list"; + + //----------------------------------------------------------------------------- // Private class used to hold options that control hdf5 i/o params. // @@ -397,23 +401,32 @@ void hdf5_ref_path_with_filename(hid_t hdf5_id, // if incompatible, incompat_details contains human readable details // about why //----------------------------------------------------------------------------- -bool check_if_conduit_leaf_is_compatible_with_hdf5_obj(const DataType &dtype, +bool check_if_conduit_leaf_is_compatible_with_hdf5_obj(const DataType &dtype, const std::string &ref_path, hid_t hdf5_id, std::string &incompat_details); - + //----------------------------------------------------------------------------- -bool check_if_conduit_object_is_compatible_with_hdf5_tree(const Node &node, +bool check_if_conduit_object_is_compatible_with_hdf5_tree(const Node &node, const std::string &ref_path, hid_t hdf5_id, std::string &incompat_details); //----------------------------------------------------------------------------- -bool check_if_conduit_node_is_compatible_with_hdf5_tree(const Node &node, +bool check_if_conduit_node_is_compatible_with_hdf5_tree(const Node &node, const std::string &ref_path, hid_t hdf5_id, std::string &incompat_details); +//----------------------------------------------------------------------------- +bool check_if_conduit_list_is_compatible_with_hdf5_tree(const Node &node, + const std::string &ref_path, + hid_t hdf5_id, + std::string &incompat_details); + +//----------------------------------------------------------------------------- +bool check_if_hdf5_group_has_conduit_list_attribute(hid_t hdf5_group_id); + //----------------------------------------------------------------------------- // helpers for writing //----------------------------------------------------------------------------- @@ -424,6 +437,12 @@ hid_t create_hdf5_dataset_for_conduit_leaf(const DataType &dt, hid_t hdf5_group_id, const std::string &hdf5_dset_name); +//----------------------------------------------------------------------------- +hid_t create_hdf5_group_for_conduit_node(const Node &node, + const std::string &ref_path, + hid_t hdf5_parent_group_id, + const std::string &hdf5_child_group_name); + //----------------------------------------------------------------------------- void write_conduit_leaf_to_hdf5_dataset(const Node &node, const std::string &ref_path, @@ -436,10 +455,17 @@ void write_conduit_leaf_to_hdf5_group(const Node &node, const std::string &hdf5_dset_name); //----------------------------------------------------------------------------- -void write_conduit_object_to_hdf5_group(const Node &node, - const std::string &ref_path, - hid_t hdf5_group_id); +void write_conduit_node_children_to_hdf5_group(const Node &node, + const std::string &ref_path, + hid_t hdf5_group_id); +//----------------------------------------------------------------------------- +void write_conduit_hdf5_list_attribute(hid_t hdf5_group_id, + const std::string &ref_path); + +//----------------------------------------------------------------------------- +void remove_conduit_hdf5_list_attribute(hid_t hdf5_group_id, + const std::string &ref_path); //----------------------------------------------------------------------------- // helpers for reading @@ -948,13 +974,6 @@ check_if_conduit_leaf_is_compatible_with_hdf5_obj(const DataType &dtype, incompat_details = oss.str(); res = false; } - // - // if(res == false) - // { - // CONDUIT_INFO("leaf in Conduit Node at path " << ref_path << - // " is not compatible with given HDF5 tree at path " - // << ref_path); - // } return res; } @@ -973,7 +992,7 @@ check_if_conduit_object_is_compatible_with_hdf5_tree(const Node &node, H5O_info_t h5_obj_info; herr_t h5_status = H5Oget_info(hdf5_id, &h5_obj_info); - // make sure it is a dataset ... + // make sure it is a group ... if( CONDUIT_HDF5_STATUS_OK(h5_status) && (h5_obj_info.type == H5O_TYPE_GROUP) ) { @@ -1027,6 +1046,96 @@ check_if_conduit_object_is_compatible_with_hdf5_tree(const Node &node, return res; } +//---------------------------------------------------------------------------// +bool +check_if_conduit_list_is_compatible_with_hdf5_tree(const Node &node, + const std::string &ref_path, + hid_t hdf5_id, + std::string &incompat_details) +{ + bool res = true; + + // make sure we have a group ... + + H5O_info_t h5_obj_info; + herr_t h5_status = H5Oget_info(hdf5_id, &h5_obj_info); + + // make sure it is a group ... + if( CONDUIT_HDF5_STATUS_OK(h5_status) && + (h5_obj_info.type == H5O_TYPE_GROUP) ) + { + // TODO: should we force the group should have our att that signals a + // list ? + + // if(!check_if_hdf5_group_has_conduit_list_attribute(hdf5_id,ref_path)) + // { + // // we don't have a list + // std::ostringstream oss; + // oss << "Conduit Node (list) at path '" << ref_path << "'" + // << " is not compatible with given HDF5 tree at path" + // << " '" << ref_path << "'" + // << "\nHDF5 Group missing '" << conduit_hdf5_list_attr_name + // << "' attribute"; + // incompat_details = oss.str(); + // res = false; + // return res; + // } + + NodeConstIterator itr = node.children(); + + // call on each child with expanded path + while(itr.has_next() && res) + { + + const Node &child = itr.next(); + // check if the HDF5 group has ordered children + // that are compat with our node. + + // for a list, we check the group's children by index (not name) + + hid_t h5_child_obj = H5Oopen_by_idx(hdf5_id, ".", + H5_INDEX_CRT_ORDER, + H5_ITER_INC, + itr.index(), + H5P_DEFAULT); + + std::string chld_ref_path = join_ref_paths(ref_path,itr.name()); + if( CONDUIT_HDF5_VALID_ID(h5_child_obj) ) + { + // if a child does exist, we need to make sure the child is + // compatible with the conduit node + res = check_if_conduit_node_is_compatible_with_hdf5_tree(child, + chld_ref_path, + h5_child_obj, + incompat_details); + + CONDUIT_CHECK_HDF5_ERROR_WITH_FILE_AND_REF_PATH(H5Oclose(h5_child_obj), + hdf5_id, + ref_path, + "Failed to close HDF5 Object: " << h5_child_obj); + } + // no child exists with this index, we are ok (it can be created + // to match) + } + } + else // bad id or not a group + { + std::ostringstream oss; + oss << "Conduit Node (list) at path '" << ref_path << "'" + << " is not compatible with given HDF5 tree at path" + << "'" << ref_path << "'" + << "\nConduit List vs HDF5 Group: Bad HDF5 Group ID " + << "or HDF5 ID is not a HDF5 Group"; + + incompat_details = oss.str(); + + res = false; + } + + return res; +} + + //---------------------------------------------------------------------------// bool @@ -1053,6 +1162,13 @@ check_if_conduit_node_is_compatible_with_hdf5_tree(const Node &node, hdf5_id, incompat_details); } + else if(dt.is_list()) + { + res = check_if_conduit_list_is_compatible_with_hdf5_tree(node, + ref_path, + hdf5_id, + incompat_details); + } else // not supported { std::ostringstream oss; @@ -1061,13 +1177,6 @@ check_if_conduit_node_is_compatible_with_hdf5_tree(const Node &node, << " for HDF5 i/o and cannot be written to HDF5 path" << " '" << ref_path << "'"; - if(dt.is_list()) // let them know there is hope for lists in the future - { - // we don't support lists yet, but provide helpful - // info when passed a list - oss << "\nA future conduit release will add" - << " list read/write support for HDF5."; - } incompat_details = oss.str(); res = false; } @@ -1075,6 +1184,29 @@ check_if_conduit_node_is_compatible_with_hdf5_tree(const Node &node, return res; } +//---------------------------------------------------------------------------// +bool +check_if_hdf5_group_has_conduit_list_attribute(hid_t hdf5_group_id, + const std::string &ref_path) +{ + htri_t h5_att_status = H5Aexists_by_name(hdf5_group_id, ".", + conduit_hdf5_list_attr_name.c_str(), + H5P_DEFAULT); + // + // https://support.hdfgroup.org/HDF5/doc/RM/RM_H5A.html#Annot-ExistsByName + // > 0 exists, 0 doesn't exist, < 0 error + // + + CONDUIT_CHECK_HDF5_ERROR_WITH_REF_PATH(h5_att_status, + ref_path, + "Failed call to H5Aexists_by_name" + << " to check for '" + << conduit_hdf5_list_attr_name + << "' attribute of HDF5 Group ID " + << " " << hdf5_group_id); + return h5_att_status > 0; +} + //---------------------------------------------------------------------------// hid_t create_hdf5_compact_plist_for_conduit_leaf() @@ -1256,7 +1388,92 @@ create_hdf5_dataset_for_conduit_empty(hid_t hdf5_group_id, return res; } +//----------------------------------------------------------------------------- +hid_t +create_hdf5_group_for_conduit_node(const Node &node, + const std::string &ref_path, + hid_t hdf5_parent_group_id, + const std::string &hdf5_new_group_name) +{ + hid_t h5_gc_plist = H5Pcreate(H5P_GROUP_CREATE); + + CONDUIT_CHECK_HDF5_ERROR_WITH_FILE_AND_REF_PATH(h5_gc_plist, + hdf5_parent_group_id, + ref_path, + "Failed to create H5P_GROUP_CREATE property " + << " list"); + + // track creation order + herr_t h5_status = H5Pset_link_creation_order(h5_gc_plist, + ( H5P_CRT_ORDER_TRACKED | H5P_CRT_ORDER_INDEXED) ); + + CONDUIT_CHECK_HDF5_ERROR_WITH_FILE_AND_REF_PATH(h5_status, + hdf5_parent_group_id, + ref_path, + "Failed to set group link creation property"); + + // prefer compact group storage + // https://support.hdfgroup.org/HDF5/doc/RM/RM_H5G.html#Group-GroupStyles + h5_status = H5Pset_link_phase_change(h5_gc_plist, + 32, // max for compact storage + 32); // min for dense storage + + CONDUIT_CHECK_HDF5_ERROR_WITH_FILE_AND_REF_PATH(h5_status, + hdf5_parent_group_id, + ref_path, + "Failed to set group link phase change property "); + + // calc hints for meta data about link names + NodeConstIterator chld_itr = node.children(); + + index_t chld_names_avg_size = 0; + index_t num_children = node.number_of_children(); + + while(chld_itr.has_next()) + { + chld_itr.next(); + chld_names_avg_size +=chld_itr.name().size(); + } + + if(chld_names_avg_size > 0 && num_children > 0 ) + { + chld_names_avg_size = chld_names_avg_size / num_children; + } + + // set hints for meta data about link names + h5_status = H5Pset_est_link_info(h5_gc_plist, + // number of children + (unsigned int)num_children, + // est name size + (unsigned int)chld_names_avg_size); + + CONDUIT_CHECK_HDF5_ERROR_WITH_FILE_AND_REF_PATH(h5_status, + hdf5_parent_group_id, + ref_path, + "Failed to set group est link info property "); + + hid_t h5_child_id = H5Gcreate(hdf5_parent_group_id, + hdf5_new_group_name.c_str(), + H5P_DEFAULT, + h5_gc_plist, + H5P_DEFAULT); + + CONDUIT_CHECK_HDF5_ERROR_WITH_FILE_AND_REF_PATH(h5_child_id, + hdf5_parent_group_id, + ref_path, + "Failed to create HDF5 Group " + << " parent: " << hdf5_parent_group_id + << " name: " << hdf5_new_group_name); + + CONDUIT_CHECK_HDF5_ERROR_WITH_FILE_AND_REF_PATH(H5Pclose(h5_gc_plist), + hdf5_parent_group_id, + ref_path, + "Failed to close HDF5 H5P_GROUP_CREATE " + << "property list: " + << h5_gc_plist); + return h5_child_id; +} //---------------------------------------------------------------------------// @@ -1304,6 +1521,8 @@ write_conduit_leaf_to_hdf5_dataset(const Node &node, conduit_dtype_to_hdf5_dtype_cleanup(h5_dtype_id); } + + //---------------------------------------------------------------------------// void write_conduit_leaf_to_hdf5_group(const Node &node, @@ -1417,14 +1636,43 @@ write_conduit_empty_to_hdf5_group(hid_t hdf5_group_id, } +//---------------------------------------------------------------------------// +void +setup_hdf5_group_atts_for_conduit_node(const Node &node, + const std::string &ref_path, + hid_t hdf5_group_id) +{ + bool has_list_attr = check_if_hdf5_group_has_conduit_list_attribute(hdf5_group_id, + ref_path); + + if( !has_list_attr && node.dtype().is_list() ) + { + write_conduit_hdf5_list_attribute(hdf5_group_id, + ref_path); + } + + if( has_list_attr && node.dtype().is_object() ) + { + std::cout << " remove_conduit_hdf5_list_attribute " << std::endl; + remove_conduit_hdf5_list_attribute(hdf5_group_id, + ref_path); + } +} + + //---------------------------------------------------------------------------// // assume this is called only if we know the hdf5 state is compatible //---------------------------------------------------------------------------// void -write_conduit_object_to_hdf5_group(const Node &node, - const std::string &ref_path, - hid_t hdf5_group_id) +write_conduit_node_children_to_hdf5_group(const Node &node, + const std::string &ref_path, + hid_t hdf5_group_id) { + // make sure our special atts are setup correctly + setup_hdf5_group_atts_for_conduit_node(node, + ref_path, + hdf5_group_id); + NodeConstIterator itr = node.children(); // call on each child with expanded path @@ -1432,13 +1680,14 @@ write_conduit_object_to_hdf5_group(const Node &node, { const Node &child = itr.next(); DataType dt = child.dtype(); + std::string child_name = itr.name(); if(dt.is_number() || dt. is_string()) { write_conduit_leaf_to_hdf5_group(child, ref_path, hdf5_group_id, - itr.name().c_str()); + child_name.c_str()); } else if(dt.is_empty()) { @@ -1446,15 +1695,15 @@ write_conduit_object_to_hdf5_group(const Node &node, // a dataset with an null shape write_conduit_empty_to_hdf5_group(hdf5_group_id, ref_path, - itr.name().c_str()); + child_name.c_str()); } - else if(dt.is_object()) + else if( dt.is_object() || dt.is_list() ) { - // check if the HDF5 group has child with same name + // check if the HDF5 group has child with same name // as the node's child H5O_info_t h5_obj_info; herr_t h5_info_status = H5Oget_info_by_name(hdf5_group_id, - itr.name().c_str(), + child_name.c_str(), &h5_obj_info, H5P_DEFAULT); @@ -1464,7 +1713,7 @@ write_conduit_object_to_hdf5_group(const Node &node, { // if the hdf5 group exists, open it h5_child_id = H5Gopen(hdf5_group_id, - itr.name().c_str(), + child_name.c_str(), H5P_DEFAULT); CONDUIT_CHECK_HDF5_ERROR_WITH_FILE_AND_REF_PATH(h5_child_id, @@ -1477,89 +1726,17 @@ write_conduit_object_to_hdf5_group(const Node &node, else { // if the hdf5 group doesn't exist, we need to create it - hid_t h5_gc_plist = H5Pcreate(H5P_GROUP_CREATE); - - CONDUIT_CHECK_HDF5_ERROR_WITH_FILE_AND_REF_PATH(h5_gc_plist, - hdf5_group_id, - ref_path, - "Failed to create H5P_GROUP_CREATE property " - << " list"); - - // track creation order - herr_t h5_status = H5Pset_link_creation_order(h5_gc_plist, - ( H5P_CRT_ORDER_TRACKED | H5P_CRT_ORDER_INDEXED) ); - - CONDUIT_CHECK_HDF5_ERROR_WITH_FILE_AND_REF_PATH(h5_status, - hdf5_group_id, - ref_path, - "Failed to set group link creation property"); - - // prefer compact group storage - // https://support.hdfgroup.org/HDF5/doc/RM/RM_H5G.html#Group-GroupStyles - h5_status = H5Pset_link_phase_change(h5_gc_plist, - 32, // max for compact storage - 32); // min for dense storage - - CONDUIT_CHECK_HDF5_ERROR_WITH_FILE_AND_REF_PATH(h5_status, - hdf5_group_id, - ref_path, - "Failed to set group link phase change property "); + h5_child_id = create_hdf5_group_for_conduit_node(child, + ref_path, + hdf5_group_id, + child_name); - // calc hints for meta data about link names - NodeConstIterator chld_itr = itr.node().children(); - - index_t chld_names_avg_size = 0; - index_t num_children = itr.node().number_of_children(); - - while(chld_itr.has_next()) - { - chld_itr.next(); - chld_names_avg_size +=chld_itr.name().size(); - } - - if(chld_names_avg_size > 0 && num_children > 0 ) - { - chld_names_avg_size = chld_names_avg_size / num_children; - } - - // set hints for meta data about link names - h5_status = H5Pset_est_link_info(h5_gc_plist, - // number of children - (unsigned int)num_children, - // est name size - (unsigned int)chld_names_avg_size); - - CONDUIT_CHECK_HDF5_ERROR_WITH_FILE_AND_REF_PATH(h5_status, - hdf5_group_id, - ref_path, - "Failed to set group est link info property "); - - h5_child_id = H5Gcreate(hdf5_group_id, - itr.name().c_str(), - H5P_DEFAULT, - h5_gc_plist, - H5P_DEFAULT); - - CONDUIT_CHECK_HDF5_ERROR_WITH_FILE_AND_REF_PATH(h5_child_id, - hdf5_group_id, - ref_path, - "Failed to create HDF5 Group " - << " parent: " << hdf5_group_id - << " name: " << itr.name()); - - CONDUIT_CHECK_HDF5_ERROR_WITH_FILE_AND_REF_PATH(H5Pclose(h5_gc_plist), - hdf5_group_id, - ref_path, - "Failed to close HDF5 H5P_GROUP_CREATE " - << "property list: " - << h5_gc_plist); } - // traverse - write_conduit_object_to_hdf5_group(child, - ref_path, - h5_child_id); + write_conduit_node_children_to_hdf5_group(child, + ref_path, + h5_child_id); CONDUIT_CHECK_HDF5_ERROR_WITH_FILE_AND_REF_PATH(H5Gclose(h5_child_id), hdf5_group_id, @@ -1578,7 +1755,6 @@ write_conduit_object_to_hdf5_group(const Node &node, } - //---------------------------------------------------------------------------// // assumes compatible, dispatches to proper specific write //---------------------------------------------------------------------------// @@ -1590,26 +1766,106 @@ write_conduit_node_to_hdf5_tree(const Node &node, DataType dt = node.dtype(); // we support a leaf or a group - if(dt.is_number() || dt.is_string()) + if( dt.is_number() || dt.is_string() ) { write_conduit_leaf_to_hdf5_dataset(node, ref_path, hdf5_id); } - else if(dt.is_object()) + else if( dt.is_object() || dt.is_list() ) { - write_conduit_object_to_hdf5_group(node, - ref_path, - hdf5_id); + write_conduit_node_children_to_hdf5_group(node, + ref_path, + hdf5_id); } else // not supported { std::string hdf5_err_ref_path; hdf5_ref_path_with_filename(hdf5_id, ref_path, hdf5_err_ref_path); CONDUIT_HDF5_ERROR(hdf5_err_ref_path, - "HDF5 write doesn't support LIST_ID or EMPTY_ID nodes."); + "HDF5 write doesn't support EMPTY_ID nodes."); } } +//---------------------------------------------------------------------------// +void +write_conduit_hdf5_list_attribute(hid_t hdf5_group_id, + const std::string &ref_path) +{ + // We really just use the presence of the attribute, we don't need + // data associated with it. + // + // I tried to write a null att (null hdf5 dt, etc) but that didn't work. + // H5Awrite fails with message about null data. I could't find any + // examples that demoed this either -- it may not be supported. + // + // So, we write a single meaningless int as the attribute data. + // Perhaps someone could breath meaning into this int in the future + // or find a way to eliminate it. + + int att_value = 1; + + hid_t h5_dspace_id = H5Screate(H5S_SCALAR); + + hid_t h5_attr_id = H5Acreate(hdf5_group_id, + conduit_hdf5_list_attr_name.c_str(), + H5T_NATIVE_INT, + h5_dspace_id, + H5P_DEFAULT, + H5P_DEFAULT); + + CONDUIT_CHECK_HDF5_ERROR_WITH_FILE_AND_REF_PATH(h5_attr_id, + hdf5_group_id, + ref_path, + "Failed to create HDF5 Attribute " + << hdf5_group_id + << " " + << conduit_hdf5_list_attr_name.c_str()); + + + hid_t h5_status = H5Awrite(h5_attr_id, + H5T_NATIVE_INT, + &att_value); + CONDUIT_CHECK_HDF5_ERROR_WITH_FILE_AND_REF_PATH(h5_status, + hdf5_group_id, + ref_path, + "Failed to write HDF5 Attribute " + << hdf5_group_id + << " " + << conduit_hdf5_list_attr_name.c_str()); + + // close our dataspace + CONDUIT_CHECK_HDF5_ERROR_WITH_FILE_AND_REF_PATH(H5Sclose(h5_dspace_id), + hdf5_group_id, + ref_path, + "Failed to close HDF5 Dataspace " + << h5_dspace_id); + + // close our attribute + CONDUIT_CHECK_HDF5_ERROR_WITH_FILE_AND_REF_PATH(H5Aclose(h5_attr_id), + hdf5_group_id, + ref_path, + "Failed to close HDF5 Attribute " + << h5_attr_id); +} + +//---------------------------------------------------------------------------// +void +remove_conduit_hdf5_list_attribute(hid_t hdf5_group_id, + const std::string &ref_path) +{ + // cleanup group attached att, just in case a group changes roles + // and is still compatible otherwise + herr_t h5_status = H5Adelete(hdf5_group_id, + conduit_hdf5_list_attr_name.c_str()); + + CONDUIT_CHECK_HDF5_ERROR_WITH_FILE_AND_REF_PATH(h5_status, + hdf5_group_id, + ref_path, + "Failed to remove HDF5 Attribute " + << hdf5_group_id + << " " + << conduit_hdf5_list_attr_name.c_str()); +} @@ -1701,6 +1957,51 @@ h5_group_check(h5_read_opdata *od, } } + +//---------------------------------------------------------------------------// +Node * +h5l_iterate_traverse_op_func_get_child(Node &node, + const std::string &hdf5_path) +{ + Node *chld_node_ptr = NULL; + // if( h5_od->node->dtype().is_object() ) + if( node.dtype().is_object() ) + { + // execute traversal for this group + chld_node_ptr = &node.fetch(hdf5_path); + } + else if( node.dtype().is_list() ) + { + // we need the child index, use name to index for now + // not sure if it is possible to get iteration index + // from h5literate + + // Either the child already exists in conduit + // (compat case), or we need to append to add + // a new child + + // parse index for this list entry + std::istringstream iss(hdf5_path); + int child_idx = -1; + iss >> child_idx; + + if(node.number_of_children() <= child_idx ) + { + node.append(); + } + + chld_node_ptr = &node.child(child_idx); + } + else + { + // TODO Error: We should not land here, H5Literate should + // only be called on groups, which will correspond + // to either objects or lists + } + + return chld_node_ptr; +} + //---------------------------------------------------------------------------// /// Our main callback for H5Literate. /// (adapted from: h5ex_g_traverse) @@ -1726,7 +2027,6 @@ h5l_iterate_traverse_op_func(hid_t hdf5_id, /* Type conversion */ struct h5_read_opdata *h5_od = (struct h5_read_opdata*)hdf5_operator_data; - /* * Get type of the object and display its name and type. * The name of the object is passed to this function by @@ -1747,6 +2047,7 @@ h5l_iterate_traverse_op_func(hid_t hdf5_id, std::string chld_ref_path = h5_od->ref_path + std::string("/") + std::string(hdf5_path); + switch (h5_info_buf.type) { case H5O_TYPE_GROUP: @@ -1769,8 +2070,8 @@ h5l_iterate_traverse_op_func(hid_t hdf5_id, else { hid_t h5_group_id = H5Gopen(hdf5_id, - hdf5_path, - H5P_DEFAULT); + hdf5_path, + H5P_DEFAULT); CONDUIT_CHECK_HDF5_ERROR_WITH_FILE_AND_REF_PATH(h5_group_id, hdf5_id, h5_od->ref_path, @@ -1781,12 +2082,13 @@ h5l_iterate_traverse_op_func(hid_t hdf5_id, << " path:" << hdf5_path); - // execute traversal for this group - Node &chld_node = h5_od->node->fetch(hdf5_path); + Node *chld_node_ptr = h5l_iterate_traverse_op_func_get_child( + *h5_od->node, + std::string(hdf5_path)); read_hdf5_group_into_conduit_node(h5_group_id, chld_ref_path, - chld_node); + *chld_node_ptr); // close the group CONDUIT_CHECK_HDF5_ERROR_WITH_FILE_AND_REF_PATH(H5Gclose(h5_group_id), @@ -1800,7 +2102,10 @@ h5l_iterate_traverse_op_func(hid_t hdf5_id, } case H5O_TYPE_DATASET: { - Node &leaf = h5_od->node->fetch(hdf5_path); + Node *chld_node_ptr = h5l_iterate_traverse_op_func_get_child( + *h5_od->node, + std::string(hdf5_path)); + // open hdf5 dataset at path hid_t h5_dset_id = H5Dopen(hdf5_id, hdf5_path, @@ -1818,7 +2123,7 @@ h5l_iterate_traverse_op_func(hid_t hdf5_id, read_hdf5_dataset_into_conduit_node(h5_dset_id, chld_ref_path, - leaf); + *chld_node_ptr); // close the dataset CONDUIT_CHECK_HDF5_ERROR_WITH_FILE_AND_REF_PATH(H5Dclose(h5_dset_id), @@ -1845,16 +2150,24 @@ read_hdf5_group_into_conduit_node(hid_t hdf5_group_id, const std::string &ref_path, Node &dest) { - // we want to make sure this is a conduit object - // even if it doesn't have any children - dest.set(DataType::object()); - - // get info, we need to get the obj addr for cycle tracking H5O_info_t h5_info_buf; herr_t h5_status = H5Oget_info(hdf5_group_id, &h5_info_buf); + // Check if this is a list or an object case + if(check_if_hdf5_group_has_conduit_list_attribute(hdf5_group_id, + ref_path)) + { + // special att: we have a list + dest.set(DataType::list()); + } + else + { + // no special att: we have an object + dest.set(DataType::object()); + } + // setup the callback struct we will use for H5Literate struct h5_read_opdata h5_od; // setup linked list tracking that allows us to detect cycles @@ -1867,7 +2180,7 @@ read_hdf5_group_into_conduit_node(hid_t hdf5_group_id, h5_od.ref_path = ref_path; H5_index_t h5_grp_index_type = H5_INDEX_NAME; - + // check for creation order index using propertylist hid_t h5_gc_plist = H5Gget_create_plist(hdf5_group_id); diff --git a/src/tests/docs/t_conduit_docs_relay_io_hdf5_examples.cpp b/src/tests/docs/t_conduit_docs_relay_io_hdf5_examples.cpp index d21832a2f..89d3e132a 100644 --- a/src/tests/docs/t_conduit_docs_relay_io_hdf5_examples.cpp +++ b/src/tests/docs/t_conduit_docs_relay_io_hdf5_examples.cpp @@ -98,8 +98,16 @@ TEST(conduit_docs, relay_io_example_hdf5_interface_1) // hdf5 state, adding new paths is always fine. n["a/my_data"] = 3.1415; n["a/b/c"] = 144; - conduit::relay::io::hdf5_write(n,h5_id); + // lists are also supported + n["a/my_list"].append() = 42.0; + n["a/my_list"].append() = 42; + conduit::relay::io::hdf5_write(n,h5_id); + + // check if a subpath of a list exists + if(conduit::relay::io::hdf5_has_path(h5_id,"a/my_list/0")) + std::cout << "\nPath 'myoutput.hdf5:a/my_list/0' exists" << std::endl; + // Read the entire tree: n_read.reset(); conduit::relay::io::hdf5_read(h5_id,n_read); @@ -110,8 +118,7 @@ TEST(conduit_docs, relay_io_example_hdf5_interface_1) // check if a path is a hdf5 file: if(conduit::relay::io::is_hdf5_file("myoutput.hdf5")) - std::cout << "File \n'myoutput.hdf5' is a hdf5 file" << std::endl; - + std::cout << "\nFile 'myoutput.hdf5' is a hdf5 file" << std::endl; END_EXAMPLE("relay_io_example_hdf5_interface_1"); } diff --git a/src/tests/relay/t_relay_io_hdf5.cpp b/src/tests/relay/t_relay_io_hdf5.cpp index 8628606d8..47890651c 100644 --- a/src/tests/relay/t_relay_io_hdf5.cpp +++ b/src/tests/relay/t_relay_io_hdf5.cpp @@ -65,7 +65,7 @@ TEST(conduit_relay_io_hdf5, conduit_hdf5_write_read_by_file_name) uint32 b_val = 8; uint32 c_val = 13; uint32 d_val = 121; - + Node n; n["a"] = a_val; n["b"] = b_val; @@ -91,7 +91,7 @@ TEST(conduit_relay_io_hdf5, conduit_hdf5_write_read_by_file_name) Node n_load_2; // read from root of hdf5 file io::hdf5_read("tout_hdf5_wr.hdf5",n_load_2); - + EXPECT_EQ(n_load_2["myobj/a"].as_uint32(), a_val); EXPECT_EQ(n_load_2["myobj/b"].as_uint32(), b_val); EXPECT_EQ(n_load_2["myobj/c"].as_uint32(), c_val); @@ -100,26 +100,26 @@ TEST(conduit_relay_io_hdf5, conduit_hdf5_write_read_by_file_name) Node n_load_generic; // read from root of hdf5 file io::load("tout_hdf5_wr.hdf5",n_load_generic); - + EXPECT_EQ(n_load_generic["myobj/a"].as_uint32(), a_val); EXPECT_EQ(n_load_generic["myobj/b"].as_uint32(), b_val); EXPECT_EQ(n_load_generic["myobj/c"].as_uint32(), c_val); - - - - // save load from generic io interface + + + + // save load from generic io interface io::save(n_load_generic["myobj"],"tout_hdf5_wr_generic.hdf5:myobj"); n_load_generic["myobj/d"] = d_val; - + io::load_merged("tout_hdf5_wr_generic.hdf5",n_load_generic); - + EXPECT_EQ(n_load_generic["myobj/a"].as_uint32(), a_val); EXPECT_EQ(n_load_generic["myobj/b"].as_uint32(), b_val); EXPECT_EQ(n_load_generic["myobj/c"].as_uint32(), c_val); EXPECT_EQ(n_load_generic["myobj/d"].as_uint32(), d_val); - + } @@ -142,10 +142,10 @@ TEST(conduit_relay_io_hdf5, conduit_hdf5_write_read_special_paths) // write our node as a group @ "/" // make sure "/" works io::hdf5_write(n,"tout_hdf5_wr_special_paths_2.hdf5:/"); - + // make sure empty after ":" this works io::hdf5_write(n,"tout_hdf5_wr_special_paths_3.hdf5:"); - + Node n_load; io::hdf5_read("tout_hdf5_wr_special_paths_2.hdf5:/",n_load); @@ -153,7 +153,7 @@ TEST(conduit_relay_io_hdf5, conduit_hdf5_write_read_special_paths) EXPECT_EQ(n_load["b"].as_uint32(), b_val); n_load.reset(); - + io::hdf5_read("tout_hdf5_wr_special_paths_2.hdf5:/",n_load); EXPECT_EQ(n_load["a"].as_uint32(), a_val); EXPECT_EQ(n_load["b"].as_uint32(), b_val); @@ -170,7 +170,7 @@ TEST(conduit_relay_io_hdf5, conduit_hdf5_write_read_special_paths) TEST(conduit_relay_io_hdf5, conduit_hdf5_write_read_string) { uint32 a_val = 20; - + std::string s_val = "{string value!}"; Node n; @@ -184,12 +184,12 @@ TEST(conduit_relay_io_hdf5, conduit_hdf5_write_read_string) io::hdf5_write(n,"tout_hdf5_wr_string.hdf5:myobj"); Node n_out; - + io::hdf5_read("tout_hdf5_wr_string.hdf5:myobj",n_out); - + EXPECT_EQ(n_out["a"].as_uint32(), a_val); EXPECT_EQ(n_out["s"].as_string(), s_val); - + } @@ -198,9 +198,9 @@ TEST(conduit_relay_io_hdf5, conduit_hdf5_write_read_string) TEST(conduit_relay_io_hdf5, conduit_hdf5_write_read_array) { Node n_in(DataType::float64(10)); - + float64_array val_in = n_in.value(); - + for(index_t i=0;i<10;i++) { val_in[i] = 3.1415 * i; @@ -210,12 +210,12 @@ TEST(conduit_relay_io_hdf5, conduit_hdf5_write_read_array) io::hdf5_write(n_in,"tout_hdf5_wr_array.hdf5:myobj"); Node n_out; - + io::hdf5_read("tout_hdf5_wr_array.hdf5:myobj",n_out); - + float64_array val_out = n_out.value(); - - + + for(index_t i=0;i<10;i++) { EXPECT_EQ(val_in[i],val_out[i]); @@ -240,7 +240,7 @@ TEST(conduit_relay_io_hdf5, write_and_read_conduit_leaf_to_hdf5_dataset_handle) hid_t h5_dtype = H5T_NATIVE_SHORT; hsize_t num_eles = 2; - + hid_t h5_dspace_id = H5Screate_simple(1, &num_eles, NULL); @@ -261,15 +261,15 @@ TEST(conduit_relay_io_hdf5, write_and_read_conduit_leaf_to_hdf5_dataset_handle) vals[0] = -16; vals[1] = -16; - // this should succeed + // this should succeed io::hdf5_write(n,h5_dset_id); - - // this should also succeed + + // this should also succeed vals[1] = 16; - + io::hdf5_write(n,h5_dset_id); - + n.set(DataType::uint16(10)); // this should fail EXPECT_THROW(io::hdf5_write(n,h5_dset_id),Error); @@ -304,24 +304,24 @@ TEST(conduit_relay_io_hdf5, write_conduit_object_to_hdf5_group_handle) H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - - + + Node n; n["a/b"].set(DataType::int16(2)); int16_array vals = n["a/b"].value(); vals[0] =-16; vals[1] =-16; - - // this should succeed + + // this should succeed io::hdf5_write(n,h5_group_id); - + n["a/c"] = "mystring"; - - // this should also succeed + + // this should also succeed vals[1] = 16; - + io::hdf5_write(n,h5_group_id); - + n["a/b"].set(DataType::uint16(10)); // this should fail EXPECT_THROW(io::hdf5_write(n,h5_group_id),Error); @@ -334,7 +334,7 @@ TEST(conduit_relay_io_hdf5, write_conduit_object_to_hdf5_group_handle) EXPECT_EQ(-16,read_vals[0]); EXPECT_EQ(16,read_vals[1]); EXPECT_EQ("mystring",n_read["a/c"].as_string()); - + H5Gclose(h5_group_id); H5Fclose(h5_file_id); } @@ -399,18 +399,18 @@ TEST(conduit_relay_io_hdf5, conduit_hdf5_write_read_by_file_handle) // specify group name for conduit to read out via hdf5_path arg to read // call) h5_group_id = H5Gopen(h5_file_id, "sample_group_name", 0); - + Node n_load; io::hdf5_read(h5_group_id, n_load); - + status = H5Gclose(h5_group_id); status = H5Fclose(h5_file_id); EXPECT_EQ(n_load["a"].as_uint32(), a_val); EXPECT_EQ(n_load["b"].as_uint32(), b_val); EXPECT_EQ(n_load["c"].as_uint32(), c_val); - + } @@ -418,24 +418,24 @@ TEST(conduit_relay_io_hdf5, conduit_hdf5_write_read_by_file_handle) TEST(conduit_relay_io_hdf5, conduit_hdf5_write_to_existing_dset) { Node n_in(DataType::uint32(2)); - - + + uint32_array val_in = n_in.value(); - + val_in[0] = 1; val_in[1] = 2; - - + + // Set up hdf5 file and group that caller code would already have. hid_t h5_file_id = H5Fcreate("tout_hdf5_wr_existing_dset.hdf5", H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); - - + + io::hdf5_write(n_in,h5_file_id,"myarray"); - - + + val_in[0] = 3; val_in[1] = 4; @@ -447,64 +447,64 @@ TEST(conduit_relay_io_hdf5, conduit_hdf5_write_to_existing_dset) EXPECT_THROW(io::hdf5_write(n_incompat,h5_file_id,"myarray"), conduit::Error); - + H5Fclose(h5_file_id); - + // check that the second set of values are the ones we get back - + Node n_read; - + io::hdf5_read("tout_hdf5_wr_existing_dset.hdf5:myarray",n_read); - + uint32_array val = n_read.value(); - + EXPECT_EQ(val[0],3); EXPECT_EQ(val[1],4); - + Node n_w2; n_w2["myarray"].set_external(n_read); n_w2["a/b/c"].set_uint64(123); - - // this should be compatible + + // this should be compatible io::hdf5_write(n_w2,"tout_hdf5_wr_existing_dset.hdf5"); - + n_read.reset(); - + io::hdf5_read("tout_hdf5_wr_existing_dset.hdf5",n_read); - - + + uint32_array myarray_val = n_read["myarray"].value(); - + uint64 a_b_c_val = n_read["a/b/c"].value(); - + EXPECT_EQ(myarray_val[0],3); EXPECT_EQ(myarray_val[1],4); EXPECT_EQ(a_b_c_val,123); - - + + } //----------------------------------------------------------------------------- TEST(conduit_relay_io_hdf5, conduit_hdf5_write_read_leaf_arrays) { Node n; - + n["v_int8"].set(DataType::int8(5)); n["v_int16"].set(DataType::int16(5)); n["v_int32"].set(DataType::int32(5)); n["v_int64"].set(DataType::int64(5)); - + n["v_uint8"].set(DataType::uint8(5)); n["v_uint16"].set(DataType::uint16(5)); n["v_uint32"].set(DataType::uint32(5)); n["v_uint64"].set(DataType::uint64(5)); - + n["v_float32"].set(DataType::float32(5)); n["v_float64"].set(DataType::float64(5)); - + n["v_string"].set("my_string"); - - + + int8 *v_int8_ptr = n["v_int8"].value(); int16 *v_int16_ptr = n["v_int16"].value(); int32 *v_int32_ptr = n["v_int32"].value(); @@ -533,26 +533,26 @@ TEST(conduit_relay_io_hdf5, conduit_hdf5_write_read_leaf_arrays) v_float32_ptr[i] = 32.0; v_float64_ptr[i] = 64.0; } - + n.print_detailed(); - + io::hdf5_write(n,"tout_hdf5_wr_leaf_arrays.hdf5"); - - - + + + Node n_load; - + io::hdf5_read("tout_hdf5_wr_leaf_arrays.hdf5",n_load); - + n_load.print_detailed(); - - + + int8_array v_int8_out = n_load["v_int8"].value(); int16_array v_int16_out = n_load["v_int16"].value(); int32_array v_int32_out = n_load["v_int32"].value(); int64_array v_int64_out = n_load["v_int64"].value(); - + EXPECT_EQ(v_int8_out.number_of_elements(),5); EXPECT_EQ(v_int16_out.number_of_elements(),5); EXPECT_EQ(v_int32_out.number_of_elements(),5); @@ -577,7 +577,7 @@ TEST(conduit_relay_io_hdf5, conduit_hdf5_write_read_leaf_arrays) std::string v_string_out = n_load["v_string"].as_string(); - + EXPECT_EQ(v_string_out,"my_string"); } @@ -588,13 +588,13 @@ TEST(conduit_relay_io_hdf5, conduit_hdf5_write_read_empty) Node n; n["path/to/empty"]; n.print_detailed(); - + io::hdf5_write(n,"tout_hdf5_wr_empty.hdf5"); - + Node n_load; io::hdf5_read("tout_hdf5_wr_empty.hdf5",n_load); n_load.print_detailed(); - + EXPECT_EQ(n["path/to/empty"].dtype().id(), n_load["path/to/empty"].dtype().id()); } @@ -615,7 +615,7 @@ TEST(conduit_relay_io_hdf5, hdf5_write_zero_sized_leaf) Node n_load; io::hdf5_read("tout_hdf5_w_0.hdf5",n_load); n_load.print_detailed(); - + EXPECT_EQ(n_load["a"].dtype().number_of_elements(),0); EXPECT_EQ(n_load["a"].dtype().id(),DataType::FLOAT64_ID); } @@ -627,13 +627,13 @@ TEST(conduit_relay_io_hdf5, conduit_hdf5_write_read_childless_object) Node n; n["path/to/empty"].set(DataType::object()); n.print_detailed(); - + io::hdf5_write(n,"tout_hdf5_wr_cl_obj.hdf5"); - + Node n_load; io::hdf5_read("tout_hdf5_wr_cl_obj.hdf5",n_load); n_load.print_detailed(); - + EXPECT_EQ(n["path/to/empty"].dtype().id(), n_load["path/to/empty"].dtype().id()); } @@ -643,16 +643,16 @@ TEST(conduit_relay_io_hdf5, conduit_hdf5_write_read_childless_object) //----------------------------------------------------------------------------- TEST(conduit_relay_io_hdf5, conduit_hdf5_test_write_incompat) { - + Node n; n["a/b/leaf"] = DataType::uint32(2); n["a/b/grp/leaf"].set_uint32(10); - + uint32_array vals = n["a/b/leaf"].value(); - + vals[0] = 1; vals[1] = 2; - + io::hdf5_write(n,"tout_hdf5_test_write_incompat.hdf5"); n.print(); @@ -660,7 +660,7 @@ TEST(conduit_relay_io_hdf5, conduit_hdf5_test_write_incompat) Node n2; n2["a/b/leaf/v"] = DataType::float64(2); n2["a/b/grp/leaf/v"].set_float64(10.0); - + n2.print(); hid_t h5_file_id = H5Fopen("tout_hdf5_test_write_incompat.hdf5", @@ -675,7 +675,7 @@ TEST(conduit_relay_io_hdf5, conduit_hdf5_test_write_incompat) { CONDUIT_INFO(e.message()); } - + H5Fclose(h5_file_id); } @@ -687,7 +687,7 @@ TEST(conduit_relay_io_hdf5, auto_endian) Node n; n["a"].set_int64(12345689); n["b"].set_int64(-12345689); - + if(Endianness::machine_is_big_endian()) { n.endian_swap_to_little(); @@ -698,7 +698,7 @@ TEST(conduit_relay_io_hdf5, auto_endian) } io::hdf5_write(n,"tout_hdf5_wr_opp_endian.hdf5"); - + Node n_load; io::hdf5_read("tout_hdf5_wr_opp_endian.hdf5",n_load); @@ -718,14 +718,14 @@ TEST(conduit_relay_io_hdf5, hdf5_path_exists) n["a/b/c/f"] = 20; io::hdf5_write(n,test_file_name); - - + + hid_t h5_file_id = H5Fopen(test_file_name.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT); hid_t h5_grp_a = H5Gopen(h5_file_id, "a", 0); - + EXPECT_TRUE(io::hdf5_has_path(h5_file_id,"a")); EXPECT_TRUE(io::hdf5_has_path(h5_file_id,"a/b")); @@ -750,7 +750,7 @@ TEST(conduit_relay_io_hdf5, hdf5_path_exists) EXPECT_FALSE(io::hdf5_has_path(h5_grp_a,"b/c/BAD")); EXPECT_FALSE(io::hdf5_has_path(h5_grp_a,"b/c/d/e/f/g")); - + H5Gclose(h5_grp_a); H5Fclose(h5_file_id); } @@ -758,7 +758,7 @@ TEST(conduit_relay_io_hdf5, hdf5_path_exists) //----------------------------------------------------------------------------- TEST(conduit_relay_io_hdf5, hdf5_create_append_methods) -{ +{ std::string test_file_name = "tout_hdf5_open_append.hdf5"; if(utils::is_file(test_file_name)) { @@ -768,79 +768,79 @@ TEST(conduit_relay_io_hdf5, hdf5_create_append_methods) Node n; n["a/b/c/d"] = 10; - + io::hdf5_write(n,test_file_name,true); - + n.reset(); n["a/b/c/e"] = 20; - + io::hdf5_write(n,test_file_name,true); - - + + Node n_load; io::hdf5_read(test_file_name,n_load); - + EXPECT_TRUE(n_load.has_path("a")); EXPECT_TRUE(n_load.has_path("a/b")); EXPECT_TRUE(n_load.has_path("a/b/c")); EXPECT_TRUE(n_load.has_path("a/b/c/d")); EXPECT_TRUE(n_load.has_path("a/b/c/e")); - + EXPECT_EQ(n_load["a/b/c/d"].to_int32(),10); EXPECT_EQ(n_load["a/b/c/e"].to_int32(),20); - - + + io::hdf5_write(n,test_file_name,false); - + n_load.reset(); io::hdf5_read(test_file_name,n_load); - + EXPECT_FALSE(n_load.has_path("a/b/c/d")); EXPECT_EQ(n_load["a/b/c/e"].to_int32(),20); - + n.reset(); n["a/b/c/d"] = 10; io::hdf5_save(n,test_file_name); - + n_load.reset(); io::hdf5_read(test_file_name,n_load); - + EXPECT_FALSE(n_load.has_path("a/b/c/e")); EXPECT_EQ(n_load["a/b/c/d"].to_int32(),10); - + n.reset(); n["a/b/c/e"] = 20; - + io::hdf5_write(n,test_file_name,true); - + n.reset(); n["a/b/c/e"] = 20; - + io::hdf5_append(n,test_file_name); n_load.reset(); io::hdf5_read(test_file_name,n_load); - EXPECT_TRUE(n_load.has_path("a/b/c/d")); + EXPECT_TRUE(n_load.has_path("a/b/c/d")); EXPECT_TRUE(n_load.has_path("a/b/c/e")); EXPECT_EQ(n_load["a/b/c/d"].to_int32(),10); - EXPECT_EQ(n_load["a/b/c/e"].to_int32(),20); - + EXPECT_EQ(n_load["a/b/c/e"].to_int32(),20); + } //----------------------------------------------------------------------------- TEST(conduit_relay_io_hdf5, hdf5_create_open_methods) -{ +{ std::string test_file_name = "tout_hdf5_open_and_create.hdf5"; Node n; n["a/b/c/d"] = 10; - + hid_t h5_file_id = io::hdf5_create_file(test_file_name); io::hdf5_write(n,h5_file_id); io::hdf5_close_file(h5_file_id); - + h5_file_id = io::hdf5_open_file_for_read(test_file_name); EXPECT_TRUE(io::hdf5_has_path(h5_file_id,"a")); @@ -849,20 +849,20 @@ TEST(conduit_relay_io_hdf5, hdf5_create_open_methods) EXPECT_TRUE(io::hdf5_has_path(h5_file_id,"a/b/c/d")); Node n_read; io::hdf5_read(h5_file_id,"a/b/c/d",n_read); - + EXPECT_EQ(10,n_read.to_int()); - + io::hdf5_close_file(h5_file_id); - + h5_file_id = io::hdf5_open_file_for_read_write(test_file_name); - + Node n2; n2 = 12; io::hdf5_write(n2,h5_file_id,"a/b/c/e"); EXPECT_TRUE(io::hdf5_has_path(h5_file_id,"a/b/c/e")); io::hdf5_read(h5_file_id,"a/b/c/e",n_read); EXPECT_EQ(12,n_read.to_int()); - + io::hdf5_close_file(h5_file_id); } @@ -873,14 +873,14 @@ TEST(conduit_relay_io_hdf5, conduit_hdf5_save_generic_options) { // 5k zeros, should compress well, but under default // threshold size - + Node n; - n["value"] = DataType::float64(5000); - + n["value"] = DataType::float64(5000); + Node opts; opts["hdf5/chunking/threshold"] = 2000; opts["hdf5/chunking/chunk_size"] = 2000; - + std::string tout_std = "tout_hdf5_save_generic_default_options.hdf5"; std::string tout_cmp = "tout_hdf5_save_generic_test_options.hdf5"; @@ -899,12 +899,12 @@ TEST(conduit_relay_io_hdf5, conduit_hdf5_save_generic_options) EXPECT_TRUE(utils::is_file(tout_std)); EXPECT_TRUE(utils::is_file(tout_cmp)); - + int64 tout_std_fs = utils::file_size(tout_std); int64 tout_cmp_fs = utils::file_size(tout_cmp); - CONDUIT_INFO("fs test: std = " - << tout_std_fs - << ", cmp =" + CONDUIT_INFO("fs test: std = " + << tout_std_fs + << ", cmp =" << tout_cmp_fs); EXPECT_TRUE(tout_cmp_fs < tout_std_fs); } @@ -915,31 +915,31 @@ TEST(conduit_relay_io_hdf5, conduit_hdf5_save_generic_options) TEST(conduit_relay_io_hdf5, conduit_hdf5_group_list_children) { Node n; - + n["path/sub1/a"]; n["path/sub1/b"]; n["path/sub1/c"]; - + n["path/sub2/d"]; n["path/sub2/e"]; n["path/sub2/f"]; - + std::string tout = "tout_hdf5_grp_chld_names.hdf5"; if(utils::is_file(tout)) { utils::remove_file(tout); } - + io::save(n,tout, "hdf5"); EXPECT_TRUE(utils::is_file(tout)); hid_t h5_file_id = io::hdf5_open_file_for_read(tout); std::vector cnames; - + io::hdf5_group_list_child_names(h5_file_id,"/",cnames); - + EXPECT_EQ(cnames.size(),1); EXPECT_EQ(cnames[0],"path"); @@ -949,16 +949,16 @@ TEST(conduit_relay_io_hdf5, conduit_hdf5_group_list_children) EXPECT_EQ(cnames[0],"sub1"); EXPECT_EQ(cnames[1],"sub2"); - + io::hdf5_group_list_child_names(h5_file_id,"path/sub1",cnames); - + EXPECT_EQ(cnames.size(),3); EXPECT_EQ(cnames[0],"a"); EXPECT_EQ(cnames[1],"b"); EXPECT_EQ(cnames[2],"c"); - + io::hdf5_group_list_child_names(h5_file_id,"path/sub2",cnames); - + EXPECT_EQ(cnames.size(),3); EXPECT_EQ(cnames[0],"d"); EXPECT_EQ(cnames[1],"e"); @@ -971,11 +971,11 @@ TEST(conduit_relay_io_hdf5, conduit_hdf5_group_list_children) // totally bogus paths will trigger an error EXPECT_THROW(io::hdf5_group_list_child_names(h5_file_id,"this/isnt/right",cnames),Error); - + // empty string won't work in this case EXPECT_THROW(io::hdf5_group_list_child_names(h5_file_id,"",cnames),Error); - - + + } //----------------------------------------------------------------------------- @@ -989,9 +989,9 @@ TEST(conduit_relay_io_hdf5, check_if_file_is_hdf5_file) { utils::remove_file(tout); } - + io::save(n,tout, "hdf5"); - + // this should be recoged as hdf5 EXPECT_TRUE(io::is_hdf5_file(tout)); @@ -999,7 +999,7 @@ TEST(conduit_relay_io_hdf5, check_if_file_is_hdf5_file) hid_t h5_file_id = io::hdf5_open_file_for_read_write(tout); EXPECT_TRUE(io::is_hdf5_file(tout)); io::hdf5_close_file(h5_file_id); - + h5_file_id = io::hdf5_open_file_for_read(tout); EXPECT_TRUE(io::is_hdf5_file(tout)); io::hdf5_close_file(h5_file_id); @@ -1013,10 +1013,10 @@ TEST(conduit_relay_io_hdf5, check_if_file_is_hdf5_file) io::save(n,tout,"json"); - + // this should *not* be recoged as hdf5 EXPECT_FALSE(io::is_hdf5_file(tout)); - + // check totally bad path EXPECT_FALSE(io::is_hdf5_file("/path/to/somewhere/that/cant/exist")); @@ -1035,30 +1035,30 @@ TEST(conduit_relay_io_hdf5, test_remove_path) { utils::remove_file(tout); } - + io::save(n,tout, "hdf5"); - + hid_t h5_file_id = io::hdf5_open_file_for_read_write(tout); io::hdf5_remove_path(h5_file_id,"path/otherdata/leaf"); io::hdf5_close_file(h5_file_id); - + n.reset(); io::load(tout,n); EXPECT_FALSE(n.has_path("path/otherdata/leaf")); EXPECT_TRUE(n.has_path("path/otherdata")); n.print(); - - + + h5_file_id = io::hdf5_open_file_for_read_write(tout); io::hdf5_remove_path(h5_file_id,"path/otherdata"); io::hdf5_close_file(h5_file_id); - + n.reset(); io::load(tout,n); EXPECT_FALSE(n.has_path("path/otherdata")); EXPECT_TRUE(n.has_path("path")); n.print(); - + } @@ -1074,11 +1074,11 @@ TEST(conduit_relay_io_hdf5, file_name_in_error) { utils::remove_file(tout); } - + io::save(n,tout, "hdf5"); - + hid_t h5_file_id = io::hdf5_open_file_for_read_write(tout); - + Node n_read; bool had_error = false; try @@ -1112,11 +1112,11 @@ TEST(conduit_relay_io_hdf5, test_read_various_string_style) H5P_DEFAULT, H5P_DEFAULT); - // write this string several ways and make sure relay + // write this string several ways and make sure relay // can reads them all as we expect // 21 chars + null term (22 total) - + std::string my_string = "this is my {} string!"; @@ -1188,7 +1188,7 @@ TEST(conduit_relay_io_hdf5, test_read_various_string_style) std::string my_string3 = "this is my {} string! "; - + // len w/o null = 30 h5_dtype_id = H5Tcopy(H5T_C_S1); @@ -1217,11 +1217,11 @@ TEST(conduit_relay_io_hdf5, test_read_various_string_style) H5Sclose(h5_dspace_id); H5Dclose(h5_dset_id); - // temp buffer to create a null padded string as + // temp buffer to create a null padded string as // input to write to hdf5 Node n_tmp; n_tmp.set(DataType::uint8(30)); - + uint8 *mystring4_char_ptr = n_tmp.value(); // null out entire string (leave no doubt for test) for(int i=0; i < 30; i++) @@ -1268,12 +1268,12 @@ TEST(conduit_relay_io_hdf5, test_read_various_string_style) h5_dtype_id = H5Tcreate(H5T_STRING, H5T_VARIABLE); h5_dspace_id = H5Screate(H5S_SCALAR); - - + + const char *mystr_char_ptr = my_string.c_str(); - + // create new dataset - + h5_dset_id = H5Dcreate(h5_file_id, "case_5", h5_dtype_id, @@ -1294,13 +1294,13 @@ TEST(conduit_relay_io_hdf5, test_read_various_string_style) H5Dclose(h5_dset_id); H5Fclose(h5_file_id); - + // load back in and make sure we get the correct string for each case Node n_load; io::load(tout,n_load); n_load.print(); - EXPECT_EQ(n_load["case_0"].as_string(), my_string ); + EXPECT_EQ(n_load["case_0"].as_string(), my_string ); EXPECT_EQ(n_load["case_1"].as_string(), my_string ); EXPECT_EQ(n_load["case_2"].as_string(), my_string ); EXPECT_EQ(n_load["case_3"].as_string(), my_string3 ); @@ -1344,29 +1344,70 @@ TEST(conduit_relay_io_hdf5, conduit_hdf5_write_read_string_compress) } + //----------------------------------------------------------------------------- -TEST(conduit_relay_io_hdf5, conduit_hdf5_list_incompat_error_message) +TEST(conduit_relay_io_hdf5, conduit_hdf5_list) { - std::string tout_std = "tout_hdf5_w_list_error.hdf5"; + std::string tout_std = "tout_hdf5_list.hdf5"; Node n; n.append() = "42"; n.append() = "42"; n.append() = "42"; n.append() = "42"; - - bool excpt_occured = false; - try - { - io::save(n,tout_std, "hdf5"); - } - catch(Error &e) - { - excpt_occured = true; - CONDUIT_INFO(e.message()); - } - - EXPECT_TRUE(excpt_occured); + + Node &n_sub = n.append(); + n_sub.append() = 42; + n_sub.append() = 42; + n_sub.append() = 42; + n_sub.append() = 42; + n_sub.append() = 42.0; + + io::save(n,tout_std, "hdf5"); + + Node n_load, info; + io::load(tout_std,"hdf5",n_load); + + n_load.print(); + + EXPECT_FALSE(n.diff(n_load,info)); + + hid_t h5_file_id = io::hdf5_open_file_for_read(tout_std); + /// check subpath of written list + EXPECT_TRUE(io::hdf5_has_path(h5_file_id,"0")); + EXPECT_TRUE(io::hdf5_has_path(h5_file_id,"4")); + EXPECT_FALSE(io::hdf5_has_path(h5_file_id,"5")); + + EXPECT_TRUE(io::hdf5_has_path(h5_file_id,"4/0")); + EXPECT_TRUE(io::hdf5_has_path(h5_file_id,"4/4")); + EXPECT_FALSE(io::hdf5_has_path(h5_file_id,"4/5")); + + n_load.reset(); + io::hdf5_read(h5_file_id,"4/4",n_load); + EXPECT_EQ(n_load.to_float64(),42.0); + + io::hdf5_close_file(h5_file_id); + + // simple compat check (could be expanded) + Node n_check; + n_check = 42.0; + // this isn't compat with the existing file + + h5_file_id = io::hdf5_open_file_for_read_write(tout_std); + EXPECT_THROW(io::hdf5_write(n_check,h5_file_id),Error); + + // orig should be compat + n_check.set(n); + io::hdf5_write(n_check,h5_file_id); + // lets change the value of one of the list entries + n_check[4][4] = 3.1415; + io::hdf5_write(n_check,h5_file_id); + io::hdf5_close_file(h5_file_id); + + io::load(tout_std,"hdf5",n_load); + + n_load.print(); + EXPECT_FALSE(n_check.diff(n_load,info)); }