add zfp relay features and zfparray blueprint (#430)

* relay: add support for zfparrays

support to zfparray object <-> Node (zfp_read() and zfp_write())

* Add cmake/thirdparty scripts to identify installed zfp-project.

Pass in ZFP_DIR to enable ZFP support

* add zfp_read() relay test

* add relay test for zfp_write()

* relay zfp: catch exceptions elegantly

* update relay tests with exceptions

* add blueprint zfp support

* add zfp blueprint tests

* remove zfp dependency from blueprint, which entails moving zfp word size check into relay's zfp_write()

* look for ZFP in lib64 and lib sub-dirs. And fix misc syntax in tests (dynamic mem, namespaced datatype)

* rename relay zfp methods zfp_read/write() to wrap/unwrap_zfparray()

* always compile zfp-related files in blueprint because it does not require linkage to zfp

* blueprint: make zfparray available for querying at runtime

* always compile zfp blueprint tests, now that zfp is always supported in blueprint

* rewrite blueprint zfp tests to be entirely independent of relay and zfp

src/cmake/Setup3rdParty.cmake

@@ -150,3 +150,16 @@ if(ADIOS_DIR)
         message(FATAL_ERROR "ADIOS_DIR is set, but ADIOS wasn't found.")
     endif()
 endif()
+
+################################
+# Setup Zfp if available
+################################
+# Search for Zfp.
+if(ZFP_DIR)
+    include(cmake/thirdparty/SetupZfp.cmake)
+    include_directories(${ZFP_INCLUDE_DIR})
+    # if we don't find Zfp, throw a fatal error
+    if(NOT ZFP_FOUND)
+        message(FATAL_ERROR "ZFP_DIR is set, but Zfp wasn't found.")
+    endif()
+endif()

src/cmake/thirdparty/SetupZfp.cmake

@@ -0,0 +1,86 @@
+###############################################################################
+# Copyright (c) 2014-2019, Lawrence Livermore National Security, LLC.
+#
+# Produced at the Lawrence Livermore National Laboratory
+#
+# LLNL-CODE-666778
+#
+# All rights reserved.
+#
+# This file is part of Conduit.
+#
+# For details, see: http://software.llnl.gov/conduit/.
+#
+# Please also read conduit/LICENSE
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the disclaimer below.
+#
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the disclaimer (as noted below) in the
+#   documentation and/or other materials provided with the distribution.
+#
+# * Neither the name of the LLNS/LLNL nor the names of its contributors may
+#   be used to endorse or promote products derived from this software without
+#   specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL SECURITY,
+# LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES  (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+###############################################################################
+#
+# Setup Zfp
+# This file defines:
+#  ZFP_FOUND - If Zfp was found
+#  ZFP_INCLUDE_DIR - The Zfp include directories
+#  ZFP_LIB - Zfp library
+
+
+# first Check for ZFP_DIR
+
+if(NOT ZFP_DIR)
+    MESSAGE(FATAL_ERROR "Zfp support needs explicit ZFP_DIR")
+endif()
+
+find_path(ZFP_INCLUDE_DIR zfp.h
+          PATHS ${ZFP_DIR}/include
+          NO_DEFAULT_PATH
+          NO_CMAKE_ENVIRONMENT_PATH
+          NO_CMAKE_PATH
+          NO_SYSTEM_ENVIRONMENT_PATH
+          NO_CMAKE_SYSTEM_PATH)
+
+find_library(ZFP_LIB NAMES zfp
+             PATHS ${ZFP_DIR}/lib64 ${ZFP_DIR}/lib
+             NO_DEFAULT_PATH
+             NO_CMAKE_ENVIRONMENT_PATH
+             NO_CMAKE_PATH
+             NO_SYSTEM_ENVIRONMENT_PATH
+             NO_CMAKE_SYSTEM_PATH)
+
+include(FindPackageHandleStandardArgs)
+# handle the QUIETLY and REQUIRED arguments and set ZFP_FOUND to TRUE
+# if all listed variables are TRUE
+find_package_handle_standard_args(Zfp  DEFAULT_MSG
+                                  ZFP_LIB ZFP_INCLUDE_DIR)
+
+mark_as_advanced(ZFP_INCLUDE_DIR
+                 ZFP_LIB)
+
+
+blt_register_library(NAME zfp
+                     INCLUDES ${ZFP_INCLUDE_DIR}
+                     LIBRARIES ${ZFP_LIB} )

src/libs/blueprint/CMakeLists.txt

@@ -63,6 +63,7 @@ set(blueprint_headers
     conduit_blueprint_mesh_examples.hpp
     conduit_blueprint_mcarray.hpp
     conduit_blueprint_mcarray_examples.hpp
+    conduit_blueprint_zfparray.hpp
     ${CMAKE_CURRENT_BINARY_DIR}/conduit_blueprint_exports.h
 )
 
@@ -85,6 +86,7 @@ set(blueprint_sources
     conduit_blueprint_mesh_examples.cpp
     conduit_blueprint_mcarray.cpp
     conduit_blueprint_mcarray_examples.cpp
+    conduit_blueprint_zfparray.cpp
     )
 
 #

src/libs/blueprint/conduit_blueprint.cpp

@@ -93,6 +93,7 @@ about(Node &n)
     n["protocols/mesh/index"]    = "enabled";
 
     n["protocols/mcarray"]  = "enabled";
+    n["protocols/zfparray"] = "enabled";
 }
 
 //---------------------------------------------------------------------------//

src/libs/blueprint/conduit_blueprint.hpp

@@ -64,6 +64,7 @@
 #include "conduit_blueprint_mcarray.hpp"
 #include "conduit_blueprint_mcarray_examples.hpp"
 
+#include "conduit_blueprint_zfparray.hpp"
 
 //-----------------------------------------------------------------------------
 // -- begin conduit:: --

src/libs/blueprint/conduit_blueprint_zfparray.cpp

@@ -0,0 +1,162 @@
+//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~//
+// Copyright (c) 2014-2018, Lawrence Livermore National Security, LLC.
+//
+// Produced at the Lawrence Livermore National Laboratory
+//
+// LLNL-CODE-666778
+//
+// All rights reserved.
+//
+// This file is part of Conduit.
+//
+// For details, see: http://software.llnl.gov/conduit/.
+//
+// Please also read conduit/LICENSE
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+//   this list of conditions and the disclaimer below.
+//
+// * Redistributions in binary form must reproduce the above copyright notice,
+//   this list of conditions and the disclaimer (as noted below) in the
+//   documentation and/or other materials provided with the distribution.
+//
+// * Neither the name of the LLNS/LLNL nor the names of its contributors may
+//   be used to endorse or promote products derived from this software without
+//   specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL SECURITY,
+// LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE LIABLE FOR ANY
+// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES  (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+// OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+// IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~//
+
+//-----------------------------------------------------------------------------
+///
+/// file: conduit_blueprint_zfparray.cpp
+///
+//-----------------------------------------------------------------------------
+
+//-----------------------------------------------------------------------------
+// conduit includes
+//-----------------------------------------------------------------------------
+#include "conduit_blueprint_zfparray.hpp"
+#include "conduit_log.hpp"
+
+using namespace conduit;
+// Easier access to the Conduit logging functions
+using namespace conduit::utils;
+
+//-----------------------------------------------------------------------------
+// -- begin conduit:: --
+//-----------------------------------------------------------------------------
+namespace conduit
+{
+
+
+//-----------------------------------------------------------------------------
+// -- begin conduit::blueprint:: --
+//-----------------------------------------------------------------------------
+namespace blueprint
+{
+
+//-----------------------------------------------------------------------------
+// -- begin conduit::blueprint::zfparray --
+//-----------------------------------------------------------------------------
+namespace zfparray
+{
+
+//-----------------------------------------------------------------------------
+bool
+verify(const std::string &/*protocol*/,
+       const Node &/*n*/,
+       Node &info)
+{
+    // zfparray doesn't provide any nested protocols
+
+    info.reset();
+    log::validation(info, false);
+    return false;
+}
+
+
+//----------------------------------------------------------------------------
+bool verify(const conduit::Node &n,
+            Node &info)
+{
+    info.reset();
+    bool res = true;
+
+    const std::string proto_name = "zfparray";
+
+    if(!n.dtype().is_object())
+    {
+        log::error(info,proto_name, "Node has no children");
+        res = false;
+    }
+
+    // ensure header
+    if(!n.has_child(ZFP_HEADER_FIELD_NAME))
+    {
+        log::error(info, proto_name, "Node does not have zfparray header child");
+        res = false;
+    }
+    else
+    {
+        const Node &n_header = n.fetch_child(ZFP_HEADER_FIELD_NAME);
+
+        // compressed-array headers consist of uint8 words
+        if(!n_header.dtype().is_uint8()) {
+            log::error(info, proto_name, "ZFP header node's dtype is invalid");
+            res = false;
+        }
+    }
+
+    if(!n.has_child(ZFP_COMPRESSED_DATA_FIELD_NAME))
+    {
+        log::error(info, proto_name, "Node does not have zfparray compressed-data child");
+        res = false;
+    }
+    else
+    {
+        const Node &compressed_data = n.fetch_child(ZFP_COMPRESSED_DATA_FIELD_NAME);
+
+        if(!compressed_data.dtype().is_unsigned_integer()) {
+            log::error(info, proto_name, "ZFP compressed-data node's dtype is incompatible with the compiled ZFP bitstream word size");
+            res = false;
+        }
+    }
+
+    log::validation(info, res);
+
+    return res;
+}
+
+//-----------------------------------------------------------------------------
+}
+//-----------------------------------------------------------------------------
+// -- end conduit::blueprint::zfparray --
+//-----------------------------------------------------------------------------
+
+
+}
+//-----------------------------------------------------------------------------
+// -- end conduit::blueprint:: --
+//-----------------------------------------------------------------------------
+
+
+}
+//-----------------------------------------------------------------------------
+// -- end conduit:: --
+//-----------------------------------------------------------------------------