HPCC-19999 Record translator should support de-blobbing BLOB fields

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>
hpcc-systems · Jul 4, 2018 · a7bb155 · a7bb155
1 parent bd01de2
commit a7bb155
Show file tree

Hide file tree

Showing 32 changed files with 290 additions and 240 deletions.
diff --git a/common/deftype/deftype.cpp b/common/deftype/deftype.cpp
@@ -134,87 +134,6 @@ unsigned promotedIntSize[9] = { 0, 1, 2, 4, 4, 8, 8, 8, 8 };
 
 //===========================================================================
 
-
-ITypeInfo *makeType(type_t type, int size)
-{
-    switch (type)
-    {
-    case type_string:
-        return makeStringType(size, NULL, NULL);            // MORE!!
-
-    case type_varstring:
-        return makeVarStringType(size);
-
-    case type_qstring:
-        return makeQStringType(size);
-
-    case type_unicode:
-        return makeUnicodeType(size, 0);
-
-    case type_varunicode:
-        return makeVarUnicodeType(size, 0);
-
-    case type_utf8:
-        return makeUtf8Type(size, 0);
-
-    case type_data:
-        return makeDataType(size);
-
-    case type_int:
-        return makeIntType(size, false);
-
-    case type_swapint:
-        return makeSwapIntType(size, false);
-
-    case type_packedint:
-        return makePackedIntType(size, false);
-
-    case type_real:
-        return makeRealType(size);
-
-    case type_bitfield:
-        return makeBitfieldType(size);
-
-    case type_boolean:
-        return makeBoolType();
-
-    case type_blob:
-        return makeBlobType();
-
-    case type_void:
-        return makeVoidType();
-
-    case type_null:
-        return makeNullType();
-
-    case type_record:
-        return makeRecordType();
-
-    case type_pattern:
-        return makePatternType();
-
-    case type_rule:
-        return makeRuleType(NULL);
-
-    case type_token:
-        return makeTokenType();
-
-    case type_feature:
-        return makeFeatureType();
-
-    case type_event:
-        return makeEventType();
-
-    case type_any:
-        return makeAnyType();
-
-    case type_date:
-        return makeBoolType(); // JCSMORE: Todo.
-    }
-    return NULL;
-}
-
-
 static IValue * castViaString(ITypeInfo * type, size32_t len, const char * text)
 {
     Owned<IValue> temp = createStringValue(text, len);
@@ -1905,6 +1824,11 @@ static ITypeInfo * commonUpType(CHashedTypeInfo * candidate)
     return match;
 }
 
+extern DEFTYPE_API ITypeInfo *makeKeyedBlobType(ITypeInfo * basetype)
+{
+    return commonUpType(new CKeyedBlobTypeInfo(basetype));
+}
+
 extern DEFTYPE_API ITypeInfo *makeFilePosType(ITypeInfo *basetype)
 {
     assertex(basetype);

diff --git a/common/deftype/deftype.hpp b/common/deftype/deftype.hpp
@@ -177,12 +177,12 @@ extern DEFTYPE_API ITypeInfo *makeDataType(int size);
 extern DEFTYPE_API ITypeInfo *makeBitfieldType(int sizeInBits, ITypeInfo * basetype = NULL);
 extern DEFTYPE_API ITypeInfo *makeBoolType();
 extern DEFTYPE_API ITypeInfo *makeBlobType();
+extern DEFTYPE_API ITypeInfo *makeKeyedBlobType(ITypeInfo * basetype);
 extern DEFTYPE_API ITypeInfo *makeRecordType();     // not used by any IHqlExpressions
 extern DEFTYPE_API ITypeInfo *makeVoidType();
 extern DEFTYPE_API ITypeInfo *makeNullType();
 extern DEFTYPE_API ITypeInfo *makeEventType();
 extern DEFTYPE_API ITypeInfo *makeAnyType();
-extern DEFTYPE_API ITypeInfo *makeType(type_t type, int size);
 extern DEFTYPE_API IEnumeratedTypeBuilder *makeEnumeratedTypeBuilder(ITypeInfo *base, aindex_t numvalues);
 extern DEFTYPE_API ITypeInfo *makeDecimalType(unsigned digits, unsigned prec, bool isSigned);
 extern DEFTYPE_API ITypeInfo *makeDictionaryType(ITypeInfo *basetype);

diff --git a/common/deftype/deftype.ipp b/common/deftype/deftype.ipp
@@ -559,6 +559,28 @@ protected:
     Owned<ITypeInfo>    basetype;
 };
 
+class CKeyedBlobTypeInfo : public CBasedTypeInfo
+{
+public:
+
+    virtual void serialize(MemoryBuffer &tgt) { CTypeInfo::serialize(tgt); }
+public:
+    CKeyedBlobTypeInfo(ITypeInfo * _basetype) : CBasedTypeInfo(_basetype, sizeof(offset_t)) {}
+    virtual type_t getTypeCode() const { return type_blob; };
+
+    // Only used for generation of type information so no need to fully implement these
+    virtual bool isSwappedEndian()              { return false; }
+    virtual bool isInteger()                    { return true; };
+    virtual bool isScalar()                     { return true; }
+    virtual bool isSigned()                     { return false; }
+    virtual unsigned getStringLen()             { return basetype->getStringLen(); }
+    virtual unsigned getDigits()                { return 0; }
+    virtual const char *queryTypeName()         { return "blob"; }
+    virtual ITypeInfo * queryPromotedType()     { return basetype->queryPromotedType(); }
+    virtual StringBuffer &getECLType(StringBuffer & out) { return out.append("blob"); }
+    virtual unsigned getCrc()                   { return basetype->getCrc(); }
+};
+
 class CFilePosTypeInfo : public CBasedTypeInfo
 {
 public:

diff --git a/common/fileview2/fvidxsource.cpp b/common/fileview2/fvidxsource.cpp
@@ -124,7 +124,7 @@ IndexDataSource::IndexDataSource(const char * _logicalName, IHqlExpression * _di
 {
     logicalName.set(_logicalName);
     diskRecord.set(_diskRecord);
-    deserializer.setown(createRtlFieldTypeDeserializer(nullptr));
+    deserializer.setown(createRtlFieldTypeDeserializer());
     diskRecordMeta.setown(new CDynamicOutputMetaData(* static_cast<const RtlRecordTypeInfo *>(queryRtlType(*deserializer.get(), diskRecord))));
     Owned<IUserDescriptor> udesc;
     if(_username != NULL && *_username != '\0')

diff --git a/common/remote/sockfile.cpp b/common/remote/sockfile.cpp
@@ -3940,7 +3940,7 @@ static IOutputMetaData *getTypeInfoOutputMetaData(IPropertyTree &actNode, const
 {
     IPropertyTree *json = actNode.queryPropTree(typePropName);
     if (json)
-        return createTypeInfoOutputMetaData(*json, grouped, nullptr);
+        return createTypeInfoOutputMetaData(*json, grouped);
     else
     {
         StringBuffer binTypePropName(typePropName);
@@ -3949,7 +3949,7 @@ static IOutputMetaData *getTypeInfoOutputMetaData(IPropertyTree &actNode, const
             return nullptr;
         MemoryBuffer mb;
         JBASE64_Decode(jsonBin, mb);
-        return createTypeInfoOutputMetaData(mb, grouped, nullptr);
+        return createTypeInfoOutputMetaData(mb, grouped);
     }
 }
 
@@ -4021,6 +4021,10 @@ class CRemoteDiskBaseActivity : public CSimpleInterfaceOf<IRemoteActivity>, impl
     {
         throwUnexpected();
     }
+    virtual byte * lookupBlob(unsigned __int64 id) override
+    {
+        throwUnexpected();
+    }
 };
 
 class CRemoteDiskReadActivity : public CRemoteDiskBaseActivity
@@ -4214,6 +4218,10 @@ class CRemoteDiskReadActivity : public CRemoteDiskBaseActivity
     {
         return makeLocalFposOffset(partNum, prefetchBuffer.tell());
     }
+    virtual byte * lookupBlob(unsigned __int64 id) override
+    {
+        throwUnexpected();
+    }
 };
 
 

diff --git a/common/thorhelper/thorcommon.cpp b/common/thorhelper/thorcommon.cpp
@@ -2001,14 +2001,23 @@ extern THORHELPER_API IOutputMetaData *getDaliLayoutInfo(IPropertyTree const &pr
 {
     try
     {
+        Owned<IException> error;
         bool isGrouped = props.getPropBool("@grouped", false);
         if (props.hasProp("_rtlType"))
         {
             MemoryBuffer layoutBin;
             props.getPropBin("_rtlType", layoutBin);
-            return createTypeInfoOutputMetaData(layoutBin, isGrouped, nullptr);
+            try
+            {
+                return createTypeInfoOutputMetaData(layoutBin, isGrouped);
+            }
+            catch (IException *E)
+            {
+                EXCLOG(E);
+                error.setown(E); // Save to throw later if we can't recover via ECL
+            }
         }
-        else if (props.hasProp("ECL"))
+        if (props.hasProp("ECL"))
         {
             const char *kind = props.queryProp("@kind");
             bool isIndex = (kind && streq(kind, "key"));
@@ -2026,9 +2035,13 @@ extern THORHELPER_API IOutputMetaData *getDaliLayoutInfo(IPropertyTree const &pr
                 }
                 MemoryBuffer layoutBin;
                 if (exportBinaryType(layoutBin, expr, isIndex))
-                    return createTypeInfoOutputMetaData(layoutBin, isGrouped, nullptr);
+                    return createTypeInfoOutputMetaData(layoutBin, isGrouped);
             }
         }
+        if (error)
+        {
+            throw(error.getClear());
+        }
     }
     catch (IException *E)
     {

diff --git a/common/thorhelper/thorcommon.ipp b/common/thorhelper/thorcommon.ipp
@@ -721,6 +721,7 @@ class NullDiskCallback : public IThorDiskCallback, extends CInterface
     virtual unsigned __int64 getFilePosition(const void * row) { return 0; }
     virtual unsigned __int64 getLocalFilePosition(const void * row) { return 0; }
     virtual const char * queryLogicalFilename(const void * row) { return NULL; }
+    virtual const byte * lookupBlob(unsigned __int64 id) { return nullptr; }
 };
 
 extern THORHELPER_API size32_t cloneRow(ARowBuilder & rowBuilder, const void * row, IOutputMetaData * meta);

diff --git a/common/thorhelper/thorpipe.cpp b/common/thorhelper/thorpipe.cpp
@@ -255,6 +255,8 @@ class CReadRowXMLStream : implements IReadRowStream, implements IXMLSelect, impl
     virtual unsigned __int64 getFilePosition(const void * row) { return 0; }
     virtual unsigned __int64 getLocalFilePosition(const void * row) { return 0; }
     virtual const char * queryLogicalFilename(const void * row) { return ""; }
+    virtual const byte * lookupBlob(unsigned __int64 id) override { throwUnexpected(); }
+
 
 private:
     IXmlToRowTransformer * xmlTransformer;

diff --git a/ecl/hql/hqlexpr.cpp b/ecl/hql/hqlexpr.cpp
@@ -14353,7 +14353,7 @@ void exportJsonType(StringBuffer &ret, IHqlExpression *table, bool forceIndex)
     }
     else
     {
-        Owned<IRtlFieldTypeDeserializer> deserializer(createRtlFieldTypeDeserializer(nullptr));
+        Owned<IRtlFieldTypeDeserializer> deserializer(createRtlFieldTypeDeserializer());
         const RtlTypeInfo *typeInfo = buildRtlType(*deserializer.get(), table->queryType());
         dumpTypeInfo(ret, typeInfo);
     }
@@ -14372,7 +14372,7 @@ bool exportBinaryType(MemoryBuffer &ret, IHqlExpression *table, bool forceIndex)
     {
         try
         {
-            Owned<IRtlFieldTypeDeserializer> deserializer(createRtlFieldTypeDeserializer(nullptr));
+            Owned<IRtlFieldTypeDeserializer> deserializer(createRtlFieldTypeDeserializer());
             const RtlTypeInfo *typeInfo = buildRtlType(*deserializer.get(), table->queryType());
             return dumpTypeInfo(ret, typeInfo);
         }

diff --git a/ecl/hql/hqlstack.cpp b/ecl/hql/hqlstack.cpp
@@ -249,7 +249,7 @@ int FuncCallStack::push(ITypeInfo* argType, IHqlExpression* curParam)
 int FuncCallStack::pushMeta(ITypeInfo *type)
 {
     if (!deserializer)
-        deserializer.setown(createRtlFieldTypeDeserializer(nullptr));
+        deserializer.setown(createRtlFieldTypeDeserializer());
     const RtlTypeInfo *typeInfo = buildRtlType(*deserializer.get(), type);
     CDynamicOutputMetaData * meta = new CDynamicOutputMetaData(* static_cast<const RtlRecordTypeInfo *>(typeInfo));
     metas.append(*meta);

diff --git a/ecl/hql/hqlutil.cpp b/ecl/hql/hqlutil.cpp
@@ -424,7 +424,10 @@ IHqlExpression * createPhysicalIndexRecord(HqlMapTransformer & mapper, IHqlExpre
             physicalFields.append(*createPhysicalIndexRecord(mapper, cur, false, createKeyedTypes));
         else if (cur->hasAttribute(blobAtom))
         {
-            newField = createField(cur->queryId(), makeIntType(8, false), nullptr, createKeyedTypes ? createAttribute(_payload_Atom) : nullptr);
+            if (createKeyedTypes)
+                newField = createField(cur->queryId(), makeKeyedBlobType(cur->getType()), nullptr, createAttribute(_payload_Atom));
+            else
+                newField = createField(cur->queryId(), makeIntType(8, false), nullptr, nullptr);
         }
         else
         {

diff --git a/ecl/hthor/hthor.ipp b/ecl/hthor/hthor.ipp
@@ -2301,6 +2301,7 @@ public:
     virtual unsigned __int64 getFilePosition(const void * row);
     virtual unsigned __int64 getLocalFilePosition(const void * row);
     virtual const char * queryLogicalFilename(const void * row) { return logicalFileName.get(); }
+    virtual const byte * lookupBlob(unsigned __int64 id) { UNIMPLEMENTED; }
 };
 
 class CHThorBinaryDiskReadBase : public CHThorDiskReadBaseActivity, implements IIndexReadContext

diff --git a/esp/services/ws_dfu/ws_dfuService.cpp b/esp/services/ws_dfu/ws_dfuService.cpp
@@ -1641,7 +1641,7 @@ bool CWsDfuEx::onDFURecordTypeInfo(IEspContext &context, IEspDFURecordTypeInfoRe
             df->queryAttributes().getPropBin("_rtlType", layoutBin);
             if (req.getIncludeJsonTypeInfo())
             {
-                Owned<IRtlFieldTypeDeserializer> deserializer(createRtlFieldTypeDeserializer(nullptr));
+                Owned<IRtlFieldTypeDeserializer> deserializer(createRtlFieldTypeDeserializer());
                 const RtlTypeInfo *typeInfo = deserializer->deserialize(layoutBin);
                 StringBuffer jsonFormat;
                 dumpTypeInfo(jsonFormat, typeInfo);
@@ -2380,7 +2380,7 @@ void CWsDfuEx::doGetFileDetails(IEspContext &context, IUserDescriptor *udesc, co
             df->queryAttributes().getPropBin("_rtlType", layoutBin);
             if (includeJsonTypeInfo)
             {
-                Owned<IRtlFieldTypeDeserializer> deserializer(createRtlFieldTypeDeserializer(nullptr));
+                Owned<IRtlFieldTypeDeserializer> deserializer(createRtlFieldTypeDeserializer());
                 const RtlTypeInfo *typeInfo = deserializer->deserialize(layoutBin);
                 StringBuffer jsonFormat;
                 dumpTypeInfo(jsonFormat, typeInfo);

diff --git a/roxie/ccd/ccdactivities.cpp b/roxie/ccd/ccdactivities.cpp
@@ -1455,6 +1455,7 @@ class XmlRecordProcessor : public RecordProcessor, implements IXMLSelect, implem
     {
         return reader->queryLogicalFilename(row);
     }
+    virtual const byte * lookupBlob(unsigned __int64 id) { throwUnexpected(); }
 
     virtual void match(IColumnProvider &entry, offset_t startOffset, offset_t endOffset)
     {

diff --git a/roxie/ccd/ccdfile.cpp b/roxie/ccd/ccdfile.cpp
@@ -2544,10 +2544,10 @@ class CSlaveDynamicFile : public CResolvedFile
                         diskTypeInfo.append(NULL);
                         break;
                     case 1:
-                        diskTypeInfo.append(createTypeInfoOutputMetaData(serverData, false, nullptr));
+                        diskTypeInfo.append(createTypeInfoOutputMetaData(serverData, false));
                         break;
                     case 2:
-                        diskTypeInfo.append(createTypeInfoOutputMetaData(serverData, true, nullptr));
+                        diskTypeInfo.append(createTypeInfoOutputMetaData(serverData, true));
                         break;
                     case 3:
                         assertex(fileNo > 0);

diff --git a/roxie/ccd/ccdkey.cpp b/roxie/ccd/ccdkey.cpp
@@ -523,6 +523,8 @@ class CDirectReaderBase : public CInterface, implements IDirectReader, implement
     {
         deserializeSource.finishedRow();
     }
+    virtual const byte * lookupBlob(unsigned __int64 id) { throwUnexpected(); }
+
 };
 
 class InMemoryDirectReader : public CDirectReaderBase
@@ -1239,6 +1241,7 @@ class InMemoryIndexCursor : implements IDirectReader, implements ISourceRowCurso
     { 
         UNIMPLEMENTED;
     }
+    virtual const byte * lookupBlob(unsigned __int64 id) { UNIMPLEMENTED; }
 
     virtual void serializeCursorPos(MemoryBuffer &mb) const 
     {

diff --git a/roxie/ccd/ccdserver.cpp b/roxie/ccd/ccdserver.cpp
@@ -22173,6 +22173,7 @@ class CRoxieServerXmlReadActivity : public CRoxieServerDiskReadBaseActivity, imp
     {
         return reader->queryLogicalFilename(row);
     }
+    virtual const byte * lookupBlob(unsigned __int64 id) { UNIMPLEMENTED; }
 
     virtual const void *nextRow()
     {