Merge branch 'candidate-5.2.0'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

cmake_modules/optionDefaults.cmake

@@ -1,10 +1,11 @@
 
 if ( NOT PREFIX )
 if ( WIN32 )
-    if ("$ENV{ProgramFiles(x86)}" STREQUAL "")
+    set( prog86key "ProgramFiles(x86)" )
+    if ("$ENV{${prog86key}}" STREQUAL "")
         set( PREFIX "$ENV{ProgramFiles}" )
     else()
-        set( PREFIX "$ENV{ProgramFiles(x86)}" )
+        set( PREFIX "$ENV{${prog86key}}" )
     endif()
     string(REGEX REPLACE "\\\\" "/" PREFIX ${PREFIX})    
 else (WIN32)

common/fileview2/fvdsremote.cpp

@@ -408,7 +408,7 @@ void RemoteDataSourceServer::doCmdCreateWorkunit(MemoryBuffer & in, MemoryBuffer
     in.read(sequence);
     in.read(name);
 
-    DBGLOG("RemoteFileView:CreateWorkunit('%s',%d,'%s') by[%s:%"I64F"d", wuid.get(), sequence, name ? name.get() : "", username.get(), session);
+    DBGLOG("RemoteFileView:CreateWorkunit('%s',%d,'%s') by[%s:%" I64F "d", wuid.get(), sequence, name ? name.get() : "", username.get(), session);
     Owned<IConstWUResult> wuResult = resolveResult(wuid, sequence, name);
     Owned<IFvDataSource> ds = createDataSource(wuResult, wuid, username, password);
     unique_id_t id = addDataSource(session, ds);
@@ -419,7 +419,7 @@ void RemoteDataSourceServer::doCmdCreateWorkunit(MemoryBuffer & in, MemoryBuffer
     ds->queryMetaData()->serialize(out);
     out.append(ds->isIndex());
 
-    DBGLOG("RemoteFileView:CreateWorkunit returns %"I64F"d", id);
+    DBGLOG("RemoteFileView:CreateWorkunit returns %" I64F "d", id);
 }
 
 void RemoteDataSourceServer::doCmdCreateFile(MemoryBuffer & in, MemoryBuffer & out)
@@ -431,7 +431,7 @@ void RemoteDataSourceServer::doCmdCreateFile(MemoryBuffer & in, MemoryBuffer & o
     in.read(username).read(password);
     in.read(logicalName);
 
-    DBGLOG("RemoteFileView:CreateFile('%s') by[%s:%"I64F"d", logicalName.get(), username.get(), session);
+    DBGLOG("RemoteFileView:CreateFile('%s') by[%s:%" I64F "d", logicalName.get(), username.get(), session);
     Owned<IFvDataSource> ds = createFileDataSource(logicalName, cluster, username, password);
     unique_id_t id = addDataSource(session, ds);
 
@@ -441,15 +441,15 @@ void RemoteDataSourceServer::doCmdCreateFile(MemoryBuffer & in, MemoryBuffer & o
     ds->queryMetaData()->serialize(out);
     out.append(ds->isIndex());
 
-    DBGLOG("RemoteFileView:CreateFile returns %"I64F"d", id);
+    DBGLOG("RemoteFileView:CreateFile returns %" I64F "d", id);
 }
 
 void RemoteDataSourceServer::doCmdDestroy(MemoryBuffer & in, MemoryBuffer & out)
 {
     unique_id_t id;
     in.read(id);
 
-    DBGLOG("RemoteFileView:Destroy(%"I64F"d)", id);
+    DBGLOG("RemoteFileView:Destroy(%" I64F "d)", id);
     CriticalBlock block(cs);
     ForEachItemIn(idx, entries)
     {
@@ -503,7 +503,7 @@ void RemoteDataSourceServer::removeSession(SessionId id)
         RemoteDataEntry & cur = entries.item(idx);
         if (cur.session == id)
         {
-            DBGLOG("RemoteFileView:Instance Died %"I64F"d", cur.id);
+            DBGLOG("RemoteFileView:Instance Died %" I64F "d", cur.id);
             entries.remove(idx);
         }
     }

common/remote/sockfile.cpp

@@ -2458,7 +2458,7 @@ class CRemoteFileIO : public CInterface, implements IFileIO
                 EXCLOG(e,"CRemoteFileIO::write");
                 if (++tries>3)
                     throw;
-                WARNLOG("Retrying write(%"I64F"d,%d) of %s (%d)",pos,len,parent->queryLocalName(),tries);
+                WARNLOG("Retrying write(%" I64F "d,%d) of %s (%d)",pos,len,parent->queryLocalName(),tries);
                 Owned<IException> exc = e;
                 if (!reopen())
                     throw exc.getClear();
@@ -3597,7 +3597,7 @@ class CRemoteFileServer : public CInterface, implements IRemoteFileServer, imple
             return false;
         }
         if (TF_TRACE)
-            PROGLOG("read file,  handle = %d, pos = %"I64F"d, toread = %d, read = %d",handle,pos,len,numRead);
+            PROGLOG("read file,  handle = %d, pos = %" I64F "d, toread = %d, read = %d",handle,pos,len,numRead);
         {
             reply.setLength(posOfLength + sizeof(numRead) + numRead);
             reply.writeEndianDirect(posOfLength,sizeof(numRead),&numRead);
@@ -3616,7 +3616,7 @@ class CRemoteFileServer : public CInterface, implements IRemoteFileServer, imple
         __int64 size = fileio->size();
         reply.append((unsigned)RFEnoerror).append(size);
         if (TF_TRACE)
-            PROGLOG("size file,  handle = %d, size = %"I64F"d",handle,size);
+            PROGLOG("size file,  handle = %d, size = %" I64F "d",handle,size);
         return true;
     }
 
@@ -3627,7 +3627,7 @@ class CRemoteFileServer : public CInterface, implements IRemoteFileServer, imple
         msg.read(handle).read(size);
         IFileIO *fileio;
         if (TF_TRACE)
-            PROGLOG("set size file,  handle = %d, size = %"I64F"d",handle,size);
+            PROGLOG("set size file,  handle = %d, size = %" I64F "d",handle,size);
         if (!checkFileIOHandle(reply, handle, fileio))
             return false;
         fileio->setSize(size);
@@ -3824,7 +3824,7 @@ class CRemoteFileServer : public CInterface, implements IRemoteFileServer, imple
             Owned<IFile> file = createIFile(srcname.get());
             __int64 written = fileio->appendFile(file,pos,len);
             if (TF_TRACE)
-                PROGLOG("append file,  handle = %d, file=%s, pos = %"I64F"d len = %"I64F"d written = %"I64F"d",handle,srcname.get(),pos,len,written);
+                PROGLOG("append file,  handle = %d, file=%s, pos = %" I64F "d len = %" I64F "d written = %" I64F "d",handle,srcname.get(),pos,len,written);
             reply.append((unsigned)RFEnoerror).append(written);
             return true;
         }

common/thorhelper/thorxmlread.cpp

@@ -1607,8 +1607,9 @@ class CXMLParse : public CInterface, implements IXMLParse
     PTreeReaderOptions xmlOptions;
     bool step, contentRequired, isJson;
 
-    class CXMLMaker : public CInterface, implements IPTreeMaker
+    class CMakerBase : public CInterface, implements IPTreeMaker
     {
+    protected:
         CXPath xpath;
         IXMLSelect *iXMLSelect;   // NOTE - not linked - creates circular links
         CICopyArrayOf<CParseStackInfo> stack, freeParseInfo;
@@ -1624,14 +1625,14 @@ class CXMLParse : public CInterface, implements IXMLParse
     public:
         IMPLEMENT_IINTERFACE;
 
-        CXMLMaker(const char *_xpath, IXMLSelect &_iXMLSelect, bool _contentRequired, bool ignoreNameSpaces) : xpath(_xpath, ignoreNameSpaces), iXMLSelect(&_iXMLSelect), contentRequired(_contentRequired)
+        CMakerBase(const char *_xpath, IXMLSelect &_iXMLSelect, bool _contentRequired, bool ignoreNameSpaces) : xpath(_xpath, ignoreNameSpaces), iXMLSelect(&_iXMLSelect), contentRequired(_contentRequired)
         {
             lastMatchKeptLevel = 0;
             lastMatchKeptNode = lastMatchKeptNodeParent = NULL;
             maker = NULL;
             utf8Translator = NULL;
         }
-        ~CXMLMaker()
+        ~CMakerBase()
         {
             ForEachItemIn(i, stack)
                 delete &stack.item(i);
@@ -1898,6 +1899,54 @@ class CXMLParse : public CInterface, implements IXMLParse
         }
     } *iXMLMaker;
 
+    class CXMLMaker : public CMakerBase
+    {
+    public:
+        CXMLMaker(const char *_xpath, IXMLSelect &_iXMLSelect, bool _contentRequired, bool ignoreNameSpaces) : CMakerBase(_xpath, _iXMLSelect, _contentRequired, ignoreNameSpaces)
+        {
+        }
+    };
+
+    class CJSONMaker : public CMakerBase
+    {
+    public:
+        CJSONMaker(const char *_xpath, IXMLSelect &_iXMLSelect, bool _contentRequired, bool ignoreNameSpaces) : CMakerBase(_xpath, _iXMLSelect, _contentRequired, ignoreNameSpaces)
+        {
+        }
+
+        bool checkSkipRoot(const char *tag)
+        {
+            if (stack.ordinality()) //root level only
+                return false;
+            if (streq(tag, "__array__")) //xpath starts after root array
+                return true;
+            if (streq(tag, "__object__") && xpath.queryDepth()) //empty xpath matches start object, otherwise skip, xpath starts immediately after
+                return true;
+            return false;
+        }
+
+        virtual void beginNode(const char *tag, offset_t startOffset)
+        {
+            if (!checkSkipRoot(tag))
+                CMakerBase::beginNode(tag, startOffset);
+        }
+        virtual void newAttribute(const char *tag, const char *value)
+        {
+            if (stack.ordinality() && stack.tos().keep)
+                maker->newAttribute(tag, value);
+        }
+        virtual void beginNodeContent(const char *tag)
+        {
+            if (!checkSkipRoot(tag))
+                CMakerBase::beginNodeContent(tag);
+        }
+        virtual void endNode(const char *tag, unsigned length, const void *value, bool binary, offset_t endOffset)
+        {
+            if (!checkSkipRoot(tag))
+                CMakerBase::endNode(tag, length, value, binary, endOffset);
+        }
+    };
+
 public:
     IMPLEMENT_IINTERFACE;
 
@@ -1912,11 +1961,17 @@ class CXMLParse : public CInterface, implements IXMLParse
         ::Release(iXMLMaker);
         ::Release(xmlReader);
     }
+    CMakerBase *createMaker()
+    {
+        bool ignoreNameSpaces = 0 != ((unsigned)xmlOptions & (unsigned)ptr_ignoreNameSpaces);
+        if (isJson)
+            return new CJSONMaker(xpath, *iXMLSelect, contentRequired, ignoreNameSpaces);
+        return new CXMLMaker(xpath, *iXMLSelect, contentRequired, ignoreNameSpaces);
+    }
     void init()
     {
         xmlReader = NULL;
-        bool ignoreNameSpaces = 0 != ((unsigned)xmlOptions & (unsigned)ptr_ignoreNameSpaces);
-        iXMLMaker = new CXMLMaker(xpath, *iXMLSelect, contentRequired, ignoreNameSpaces);
+        iXMLMaker = createMaker();
         iXMLMaker->init();
     }
 

common/workunit/workunit.cpp

@@ -3848,7 +3848,7 @@ void CLocalWorkUnit::checkAgentRunning(WUState & state)
                     state = WUStateFailed;
                 else
                     return;
-                WARNLOG("checkAgentRunning terminated: %"I64F"d state = %d",(__int64)agent,(int)state);
+                WARNLOG("checkAgentRunning terminated: %" I64F "d state = %d",(__int64)agent,(int)state);
                 Owned<IWorkUnit> w = &lock();
                 w->setState(state);
                 Owned<IWUException> e = w->createException();
@@ -9206,7 +9206,7 @@ static WUState _waitForWorkUnit(const char * wuid, unsigned timeout, bool compil
                             default:
                                 return ret;
                         }
-                        WARNLOG("_waitForWorkUnit terminated: %"I64F"d state = %d",(__int64)agent,(int)ret);
+                        WARNLOG("_waitForWorkUnit terminated: %" I64F "d state = %d",(__int64)agent,(int)ret);
                         Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
                         Owned<IWorkUnit> wu = factory->updateWorkUnit(wuid);
                         wu->setState(ret);

common/workunit/wujobq.cpp

@@ -1667,7 +1667,7 @@ class CJobQueue: public CJobQueueBase, implements IJobQueue
                 item = qd.root->queryPropTree(path.str());
                 if (!item)
                     break;
-//              PROGLOG("WUJOBQ: Removing %s as session %"I64F"x not active",item->queryProp("@wuid"),item->getPropInt64("@session"));
+//              PROGLOG("WUJOBQ: Removing %s as session %" I64F "x not active",item->queryProp("@wuid"),item->getPropInt64("@session"));
                 removeItem(qd,item,false);
             }
             else

dali/base/dacsds.cpp

@@ -45,7 +45,7 @@ static CriticalSection SDScrit;
 #define CHECK_CONNECTED(XSTR)                                                                                        \
     if (!connected)                                                                                                   \
     {                                                                                                               \
-        LOG(MCerror, unknownJob, XSTR": Closed connection (xpath=%s, sessionId=%"I64F"d)", xpath.get(), sessionId);       \
+        LOG(MCerror, unknownJob, XSTR": Closed connection (xpath=%s, sessionId=%" I64F "d)", xpath.get(), sessionId);       \
         return;                                                                                                     \
     }
 
@@ -526,7 +526,7 @@ IPropertyTreeIterator *CRemoteConnection::doGetElements(CClientRemoteTree *tree,
 IPropertyTreeIterator *CRemoteConnection::getElements(const char *xpath, IPTIteratorCodes flags)
 {
     if (!serverIterAvailable)
-        throw MakeSDSException(SDSExcpt_VersionMismatch, "Server-side getElements not supported by server versions prior to "SDS_SVER_MIN_GETXPATHS_CONNECT);
+        throw MakeSDSException(SDSExcpt_VersionMismatch, "Server-side getElements not supported by server versions prior to " SDS_SVER_MIN_GETXPATHS_CONNECT);
     flags |= iptiter_remote;
     return root->getElements(xpath, flags);
 }
@@ -1693,10 +1693,10 @@ IRemoteConnections *CClientSDSManager::connect(IMultipleConnector *mConnect, Ses
 {
     CDaliVersion serverVersionNeeded(MIN_MCONNECT_SVER);
     if (queryDaliServerVersion().compare(serverVersionNeeded) < 0)
-        throw MakeSDSException(SDSExcpt_VersionMismatch, "Multiple connect not supported by server versions prior to "MIN_MCONNECT_SVER);
+        throw MakeSDSException(SDSExcpt_VersionMismatch, "Multiple connect not supported by server versions prior to " MIN_MCONNECT_SVER);
 
     if (0 == id || id != myProcessSession())
-        throw MakeSDSException(SDSExcpt_InvalidSessionId, ", in multi connect, sessionid=%"I64F"x", id);
+        throw MakeSDSException(SDSExcpt_InvalidSessionId, ", in multi connect, sessionid=%" I64F "x", id);
 
     CMessageBuffer mb;
     mb.append((unsigned)DAMP_SDSCMD_MCONNECT | lazyExtFlag);
@@ -1752,7 +1752,7 @@ IRemoteConnections *CClientSDSManager::connect(IMultipleConnector *mConnect, Ses
 IRemoteConnection *CClientSDSManager::connect(const char *xpath, SessionId id, unsigned mode, unsigned timeout)
 {
     if (0 == id || id != myProcessSession())
-        throw MakeSDSException(SDSExcpt_InvalidSessionId, ", connecting to %s, sessionid=%"I64F"x", xpath, id);
+        throw MakeSDSException(SDSExcpt_InvalidSessionId, ", connecting to %s, sessionid=%" I64F "x", xpath, id);
 
     CMessageBuffer mb;
     mb.append((int)DAMP_SDSCMD_CONNECT | lazyExtFlag);

dali/base/dacsds.ipp

@@ -47,7 +47,7 @@ public:
             }
             catch (IException *e)
             {
-                VStringBuffer errMsg("beforeDispose commit connectionid=%"I64F"x, xpath=%s", connectionId, xpath.get());
+                VStringBuffer errMsg("beforeDispose commit connectionid=%" I64F "x, xpath=%s", connectionId, xpath.get());
                 EXCLOG(e, errMsg.str());
                 e->Release();
             }

dali/base/dadiags.cpp

@@ -190,7 +190,7 @@ class CDaliDiagnosticsServer: public IDaliServer, public Thread
                     bool disconnect;
                     params.read(connectionId);
                     params.read(disconnect);
-                    PROGLOG("Dalidiag request to unlock connection id: %"I64F"x", connectionId);
+                    PROGLOG("Dalidiag request to unlock connection id: %" I64F "x", connectionId);
                     StringBuffer connectionInfo;
                     bool success = querySDSServer().unlock(connectionId, disconnect, connectionInfo);
                     mb.append(success);

dali/base/danqs.cpp

@@ -767,7 +767,7 @@ class CDaliNamedQueueServer: public IDaliServer, public Thread, implements IConn
         {
             char const * branch = transactionIsGet ? "TransactionalGet" : "TransactionalPut";
             StringBuffer path;
-            path.appendf("%s[@id=\"%"I64F"d\"]", branch, transactionId);
+            path.appendf("%s[@id=\"%" I64F "d\"]", branch, transactionId);
             add = root->queryPropTree(path.str());
             if(!add)
             {
@@ -935,8 +935,8 @@ class CDaliNamedQueueServer: public IDaliServer, public Thread, implements IConn
             IPropertyTree & q = iter->query();
             StringBuffer getPath;
             StringBuffer putPath;
-            getPath.appendf("TransactionalGet[@id=\"%"I64F"d\"]", transactionId);
-            putPath.appendf("TransactionalPut[@id=\"%"I64F"d\"]", transactionId);
+            getPath.appendf("TransactionalGet[@id=\"%" I64F "d\"]", transactionId);
+            putPath.appendf("TransactionalPut[@id=\"%" I64F "d\"]", transactionId);
             IPropertyTree * xferBranch;
             IPropertyTree * removeBranch;
             if(rollback)