Merge pull request #2756 from JasonRuonanWang/table

added blosc and bzip2 to Table Engine
ornladios · Jun 8, 2021 · c678e3a · c678e3a
2 parents 7071932 + 46d43b8
commit c678e3a
Show file tree

Hide file tree

Showing 3 changed files with 38 additions and 36 deletions.
diff --git a/source/adios2/engine/table/TableWriter.cpp b/source/adios2/engine/table/TableWriter.cpp
@@ -9,6 +9,7 @@
  */
 
 #include "TableWriter.tcc"
+#include "adios2/helper/adiosString.h"
 
 namespace adios2
 {
@@ -25,17 +26,17 @@ TableWriter::TableWriter(IO &io, const std::string &name, const Mode mode,
 {
     m_MpiRank = m_Comm.Rank();
     m_MpiSize = m_Comm.Size();
-
+    helper::GetParameter(m_IO.m_Parameters, "Compressor", m_UseCompressor);
     m_SubEngine = &m_SubIO.Open(m_Name, adios2::Mode::Write);
 }
 
 TableWriter::~TableWriter()
 {
-    if (m_SzOperator)
+    if (m_Compressor)
     {
-        delete m_SzOperator;
+        delete m_Compressor;
     }
-    m_SzOperator = nullptr;
+    m_Compressor = nullptr;
 }
 
 StepStatus TableWriter::BeginStep(StepMode mode, const float timeoutSeconds)

diff --git a/source/adios2/engine/table/TableWriter.h b/source/adios2/engine/table/TableWriter.h
@@ -15,11 +15,6 @@
 #include "adios2/core/Engine.h"
 #include "adios2/core/IO.h"
 #include "adios2/core/Variable.h"
-#include "adios2/helper/adiosComm.h"
-
-#ifdef ADIOS2_HAVE_SZ
-#include "adios2/operator/compress/CompressSZ.h"
-#endif
 
 namespace adios2
 {
@@ -50,8 +45,9 @@ class TableWriter : public Engine
     ADIOS m_SubAdios;
     IO &m_SubIO;
     Engine *m_SubEngine = nullptr;
-    Operator *m_SzOperator = nullptr;
-    std::unordered_map<std::string, bool> m_Indexing;
+    Operator *m_Compressor = nullptr;
+    std::unordered_map<std::string, bool> m_IndexerMap;
+    std::string m_UseCompressor;
 
     void PutSubEngine(bool finalPut = false);
 

diff --git a/source/adios2/engine/table/TableWriter.tcc b/source/adios2/engine/table/TableWriter.tcc
@@ -13,6 +13,14 @@
 
 #include "TableWriter.h"
 
+#ifdef ADIOS2_HAVE_BLOSC
+#include "adios2/operator/compress/CompressBlosc.h"
+#endif
+
+#ifdef ADIOS2_HAVE_BZIP2
+#include "adios2/operator/compress/CompressBZIP2.h"
+#endif
+
 namespace adios2
 {
 namespace core
@@ -60,36 +68,33 @@ void TableWriter::PutDeferredCommon(Variable<T> &variable, const T *data)
     if (!var)
     {
         var = &m_SubIO.DefineVariable<T>(variable.m_Name, variable.m_Shape);
-        for (const auto &i : m_IO.m_TransportsParameters)
+        if (m_UseCompressor == "blosc")
         {
-            auto itVar = i.find("Variable");
-            if (itVar != i.end() && itVar->second == variable.m_Name)
-            {
-                auto itAccuracy = i.find("Accuracy");
-                if (itAccuracy != i.end())
-                {
-#ifdef ADIOS2_HAVE_SZ
-                    m_SzOperator = new compress::CompressSZ(Params());
-                    var->AddOperation(
-                        *m_SzOperator,
-                        {{adios2::ops::sz::key::accuracy, itAccuracy->second}});
+#ifdef ADIOS2_HAVE_BLOSC
+            m_Compressor = new compress::CompressBlosc({});
+            var->AddOperation(*m_Compressor, {});
 #else
-                    std::cerr << "ADIOS2 is not compiled with SZ" << std::endl;
+            std::cerr
+                << "ADIOS2 is not compiled with c-blosc "
+                   "(https://github.com/Blosc/c-blosc), compressor not added"
+                << std::endl;
+#endif
+        }
+        else if (m_UseCompressor == "bzip2")
+        {
+#ifdef ADIOS2_HAVE_BZIP2
+            m_Compressor = new compress::CompressBZIP2({});
+            var->AddOperation(*m_Compressor, {});
+#else
+            std::cerr << "ADIOS2 is not compiled with Bzip2 "
+                         "(https://gitlab.com/federicomenaquintero/bzip2), "
+                         "compressor not added"
+                      << std::endl;
 #endif
-                }
-                auto itIndexing = i.find("Index");
-                if (itIndexing != i.end())
-                {
-                    m_Indexing[variable.m_Name] = true;
-                }
-                else
-                {
-                    m_Indexing[variable.m_Name] = false;
-                }
-            }
         }
     }
-    if (m_Indexing[variable.m_Name])
+
+    if (m_IndexerMap[variable.m_Name])
     {
         // TODO: implement indexing
     }