HPCC-18467 HThor support for record translation

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/thorhelper/CMakeLists.txt

@@ -76,6 +76,8 @@ set (    SRCS
     )
 
 include_directories ( 
+         ${CMAKE_BINARY_DIR}
+         ${CMAKE_BINARY_DIR}/oss
          ./../../system/security/securesocket 
          ./../../common/remote 
          ./../../system/jhtree 
@@ -89,6 +91,7 @@ include_directories (
          ./../deftype 
          ./../workunit
          ./../../rtl/include 
+         ./../../ecl/hql
          ./../../roxie/roxiemem
          ./../../testing/unittests
          ./../../system/tbb_sm/tbb/include
@@ -102,6 +105,7 @@ install ( TARGETS thorhelper RUNTIME DESTINATION ${EXEC_DIR} LIBRARY DESTINATION
 target_link_libraries ( thorhelper 
          jlib 
          nbcd 
+         hql
          eclrtl 
          roxiemem
          deftype 

common/thorhelper/thorcommon.cpp

@@ -28,6 +28,8 @@
 #include "eclrtl.hpp"
 #include "rtlread_imp.hpp"
 #include "rtlcommon.hpp"
+#include "eclhelper_dyn.hpp"
+#include "hqlexpr.hpp"
 #include <algorithm>
 #ifdef _USE_NUMA
 #include <numa.h>
@@ -1815,3 +1817,29 @@ void bindMemoryToLocalNodes()
     numa_bitmask_free(nodes);
 #endif
 }
+
+extern THORHELPER_API IOutputMetaData *getDaliLayoutInfo(IPropertyTree const &props)
+{
+    bool isGrouped = props.getPropBool("@grouped", false);
+    if (props.hasProp("_rtlType"))
+    {
+        MemoryBuffer layoutBin;
+        props.getPropBin("_rtlType", layoutBin);
+        return createTypeInfoOutputMetaData(layoutBin, isGrouped, nullptr);
+    }
+    else if (props.hasProp("ECL"))
+    {
+        StringBuffer layoutECL;
+        props.getProp("ECL", layoutECL);
+        MultiErrorReceiver errs;
+        Owned<IHqlExpression> expr = parseQuery(layoutECL.str(), &errs);
+        if (errs.errCount() == 0)
+        {
+            MemoryBuffer layoutBin;
+            if (exportBinaryType(layoutBin, expr))
+                return createTypeInfoOutputMetaData(layoutBin, isGrouped, nullptr);
+        }
+    }
+    return nullptr;
+}
+

common/thorhelper/thorcommon.hpp

@@ -619,4 +619,6 @@ extern THORHELPER_API void setProcessAffinity(const char * cpus);
 extern THORHELPER_API void setAutoAffinity(unsigned curProcess, unsigned processPerNode, const char * optNodes);
 extern THORHELPER_API void bindMemoryToLocalNodes();
 
+extern THORHELPER_API IOutputMetaData *getDaliLayoutInfo(IPropertyTree const &props);
+
 #endif // THORHELPER_HPP

common/workunit/pkgimpl.hpp

@@ -124,12 +124,7 @@ class WORKUNIT_API CPackageNode : implements IHpccPackage, public CInterface
         const char *val = queryEnv("control:enableFieldTranslation");
         if (!val) val = queryEnv("enableFieldTranslation"); // Backward compatibility
         if (val)
-        {
-            if (strieq(val, "payload") || strToBool(val))
-                return RecordTranslationMode::Payload;
-            else
-                return RecordTranslationMode::None;
-        }
+            return getTranslationMode(val);
         else
             return getSysFieldTranslationEnabled();
     }

ecl/eclagent/agentctx.hpp

@@ -20,6 +20,7 @@
 #include "errorlist.h"
 #include "dautils.hpp"
 #include "eclhelper.hpp"
+#include "rtldynfield.hpp"
 #include "workunit.hpp"
 
 #if (ECLAGENT_ERROR_START != 5400 || ECLAGENT_ERROR_END != 5499)
@@ -86,7 +87,7 @@ struct IAgentContext : extends IGlobalCodeContext
 
     virtual ICodeContext *queryCodeContext() = 0;
 
-    virtual IConstWorkUnit *queryWorkUnit() = 0;
+    virtual IConstWorkUnit *queryWorkUnit() const = 0;
     virtual IWorkUnit *updateWorkUnit() const = 0;
     virtual void unlockWorkUnit() = 0;
 
@@ -112,6 +113,7 @@ struct IAgentContext : extends IGlobalCodeContext
 
     virtual IGroup *getHThorGroup(StringBuffer &grpnameout) = 0;
 
+    virtual RecordTranslationMode rltEnabled() const = 0;
     virtual unsigned __int64 queryStopAfter() = 0;
 
     virtual const char *queryWuid() = 0;

ecl/eclagent/eclagent.cpp

@@ -38,6 +38,7 @@
 #include "eclrtl_imp.hpp"
 #include "rtlds_imp.hpp"
 #include "rtlcommon.hpp"
+#include "rtldynfield.hpp"
 #include "workunit.hpp"
 #include "eventqueue.hpp"
 #include "schedulectrl.hpp"
@@ -700,6 +701,17 @@ void EclAgent::abort()
         activeGraph->abort();
 }
 
+RecordTranslationMode EclAgent::rltEnabled() const
+{
+    IConstWorkUnit *wu = queryWorkUnit();
+    SCMStringBuffer val;
+    if(wu->hasDebugValue("layoutTranslationEnabled"))
+        wu->getDebugValue("layoutTranslationEnabled", val);
+    else
+        wu->getDebugValue("hthorLayoutTranslationEnabled", val);
+    return getTranslationMode(val.str());
+}
+
 IConstWUResult *EclAgent::getResult(const char *name, unsigned sequence)
 {
     IConstWorkUnit *w = queryWorkUnit();
@@ -3019,7 +3031,7 @@ char * EclAgent::queryIndexMetaData(char const * lfn, char const * xpath)
     return out.detach();
 }
 
-IConstWorkUnit *EclAgent::queryWorkUnit()
+IConstWorkUnit *EclAgent::queryWorkUnit() const
 {
     return wuRead;
 }

ecl/eclagent/eclagent.ipp

@@ -23,8 +23,8 @@
 #include "deftype.hpp"
 #include "jthread.hpp"
 #include "dllserver.hpp"
+#include "rtldynfield.hpp"
 
-//#include "agentctx.hpp"
 #include "hthor.hpp"
 #include "thorxmlwrite.hpp"
 #include "workflow.hpp"
@@ -155,7 +155,7 @@ public:
     {
         ctx->reportProgress(msg, flags);
     }
-    virtual IConstWorkUnit *queryWorkUnit()
+    virtual IConstWorkUnit *queryWorkUnit() const override
     {
         return ctx->queryWorkUnit();
     }
@@ -233,6 +233,11 @@ public:
 
     virtual void updateWULogfile()                  { return ctx->updateWULogfile(); }
 
+    virtual RecordTranslationMode rltEnabled() const override
+    {
+        return ctx->rltEnabled();
+    }
+
 protected:
     IAgentContext * ctx;
 };
@@ -502,6 +507,7 @@ public:
     virtual IEngineContext *queryEngineContext() { return this; }
     virtual char *getDaliServers();
 
+    virtual RecordTranslationMode rltEnabled() const override;
     unsigned __int64 queryStopAfter() { return stopAfter; }
 
     virtual ISectionTimer * registerTimer(unsigned activityId, const char * name)
@@ -588,8 +594,8 @@ public:
     virtual const char *loadResource(unsigned id);
     virtual ICodeContext *queryCodeContext();
     virtual bool isResult(const char * name, unsigned sequence);
-    virtual unsigned getWorkflowId();// { return workflow->queryCurrentWfid(); }
-    virtual IConstWorkUnit *queryWorkUnit();  // no link
+    virtual unsigned getWorkflowId();
+    virtual IConstWorkUnit *queryWorkUnit() const override;  // no link
     virtual IWorkUnit *updateWorkUnit() const; // links
     virtual void unlockWorkUnit();      
     virtual void reloadWorkUnit();

ecl/hql/hqlgram.y

@@ -10203,6 +10203,7 @@ mode
 
 dsOption
     : OPT               {   $$.setExpr(createAttribute(optAtom)); }
+    | VOLATILE          {   $$.setExpr(createAttribute(volatileAtom)); }
     | UNSORTED          {   $$.setExpr(createAttribute(unsortedAtom)); }
     | RANDOM            {   $$.setExpr(createAttribute(randomAtom)); }
     | SEQUENTIAL        {   $$.setExpr(createAttribute(sequentialAtom)); }

ecl/hqlcpp/hqlsource.cpp

@@ -4456,6 +4456,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityXmlRead(BuildCtx & ctx, IHqlEx
     if (expr->hasAttribute(_spill_Atom)) flags.append("|TDXtemporary");
     if (expr->hasAttribute(unsortedAtom)) flags.append("|TDRunsorted");
     if (expr->hasAttribute(optAtom)) flags.append("|TDRoptional");
+    if (expr->hasAttribute(volatileAtom)) flags.append("|TDRnocrccheck");
     if (usesContents) flags.append("|TDRusexmlcontents");
     if (mode->hasAttribute(noRootAtom)) flags.append("|TDRxmlnoroot");
     if (!filename->isConstant()) flags.append("|TDXvarfilename");

ecl/hthor/CMakeLists.txt

@@ -79,7 +79,6 @@ target_link_libraries ( hthor
          jlib
          mp
          hrpc
-         hql
          remote
          dalibase
          environment