Permalink
Browse files

Merge remote-tracking branch 'origin/candidate-3.10.0' into candidate…

…-3.10.x

Conflicts:
	version.cmake

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>
  • Loading branch information...
2 parents 692993e + 5447d8a commit fcedccad168a2233e0bb7236dae3b4002bf9e058 @richardkchapman richardkchapman committed Nov 15, 2012
@@ -1 +1 @@
-set ( CPACK_RPM_PACKAGE_REQUIRES "boost141-regex, openldap, libicu, m4, libtool, xalan-c, xerces-c, gcc-c++, openssh-server, openssh-clients, expect, libarchive")
+set ( CPACK_RPM_PACKAGE_REQUIRES "boost141-regex, openldap, libicu, m4, libtool, xalan-c, xerces-c, gcc-c++, openssh-server, openssh-clients, expect, libarchive, rsync")
@@ -1 +1 @@
-set ( CPACK_RPM_PACKAGE_REQUIRES "boost-regex, openldap, libicu, m4, libtool, libxslt, libxml2, gcc-c++, openssh-server, openssh-clients, expect, libarchive")
+set ( CPACK_RPM_PACKAGE_REQUIRES "boost-regex, openldap, libicu, m4, libtool, libxslt, libxml2, gcc-c++, openssh-server, openssh-clients, expect, libarchive, rsync")
@@ -1 +1 @@
-set ( CPACK_DEBIAN_PACKAGE_DEPENDS "libboost-regex1.34.1, libicu38, libxalan110, libxerces-c28, binutils, libldap-2.4-2, openssl, zlib1g, g++, sudo, openssh-client, openssh-server, expect, libarchive")
+set ( CPACK_DEBIAN_PACKAGE_DEPENDS "libboost-regex1.34.1, libicu38, libxalan110, libxerces-c28, binutils, libldap-2.4-2, openssl, zlib1g, g++, sudo, openssh-client, openssh-server, expect, libarchive, rsync")
@@ -1 +1 @@
-set ( CPACK_DEBIAN_PACKAGE_DEPENDS "libboost-regex1.40.0, libicu42, libxalan110, libxerces-c28, binutils, libldap-2.4-2, openssl, zlib1g, g++, openssh-client, openssh-server, expect, libarchive1")
+set ( CPACK_DEBIAN_PACKAGE_DEPENDS "libboost-regex1.40.0, libicu42, libxalan110, libxerces-c28, binutils, libldap-2.4-2, openssl, zlib1g, g++, openssh-client, openssh-server, expect, libarchive1, rsync")
@@ -1 +1 @@
-set ( CPACK_DEBIAN_PACKAGE_DEPENDS "libboost-regex1.42.0, libicu44, libxalan110, libxerces-c28, binutils, libldap-2.4-2, openssl, zlib1g, g++, openssh-client, openssh-server, expect, libarchive1")
+set ( CPACK_DEBIAN_PACKAGE_DEPENDS "libboost-regex1.42.0, libicu44, libxalan110, libxerces-c28, binutils, libldap-2.4-2, openssl, zlib1g, g++, openssh-client, openssh-server, expect, libarchive1, rsync")
@@ -1 +1 @@
-set ( CPACK_DEBIAN_PACKAGE_DEPENDS "libboost-regex1.46.1, libicu44, libxalan110, libxerces-c28, binutils, libldap-2.4-2, openssl, zlib1g, g++, openssh-client, openssh-server, expect, libarchive1")
+set ( CPACK_DEBIAN_PACKAGE_DEPENDS "libboost-regex1.46.1, libicu44, libxalan110, libxerces-c28, binutils, libldap-2.4-2, openssl, zlib1g, g++, openssh-client, openssh-server, expect, libarchive1, rsync")
@@ -1 +1 @@
-set ( CPACK_DEBIAN_PACKAGE_DEPENDS "libboost-regex1.46.1, libicu48, libxalan110, libxerces-c28, binutils, libldap-2.4-2, openssl, zlib1g, g++, openssh-client, openssh-server, expect, libarchive12")
+set ( CPACK_DEBIAN_PACKAGE_DEPENDS "libboost-regex1.46.1, libicu48, libxalan110, libxerces-c28, binutils, libldap-2.4-2, openssl, zlib1g, g++, openssh-client, openssh-server, expect, libarchive12, rsync")
@@ -1 +1 @@
-set ( CPACK_DEBIAN_PACKAGE_DEPENDS "libboost-regex1.42.0, libicu44, libxalan110, libxerces-c28, binutils, libldap-2.4-2, openssl, zlib1g, g++, openssh-client, openssh-server, expect, libarchive1")
+set ( CPACK_DEBIAN_PACKAGE_DEPENDS "libboost-regex1.42.0, libicu44, libxalan110, libxerces-c28, binutils, libldap-2.4-2, openssl, zlib1g, g++, openssh-client, openssh-server, expect, libarchive1, rsync")
@@ -1 +1 @@
-set ( CPACK_RPM_PACKAGE_REQUIRES "binutils, gcc-c++, openssh, libldap-2_4-2, libicu, libboost_regex1_42_0, libxerces-c-3_0, libxalan-c110, expect, libarchive2" )
+set ( CPACK_RPM_PACKAGE_REQUIRES "binutils, gcc-c++, openssh, libldap-2_4-2, libicu, libboost_regex1_42_0, libxerces-c-3_0, libxalan-c110, expect, libarchive2, rsync" )
@@ -1 +1 @@
-set ( CPACK_RPM_PACKAGE_REQUIRES "binutils, gcc-c++, openssh, libldap-2_4-2, libicu, libboost_regex1_44_0, libxerces-c-3_0, libxalan-c110, expect, libarchive2" )
+set ( CPACK_RPM_PACKAGE_REQUIRES "binutils, gcc-c++, openssh, libldap-2_4-2, libicu, libboost_regex1_44_0, libxerces-c-3_0, libxalan-c110, expect, libarchive2, rsync" )
View
@@ -34,8 +34,10 @@
#define DEBUG_DIR "debug"
#define DEFAULT_KEEP_LASTN_STORES 1
#define MAXDELAYS 5
-static const char *deltaHeader = "<CRC>0000000000</CRC>"; // fill in later
+static const char *deltaHeader = "<CRC>0000000000</CRC><SIZE>0000000000000000</SIZE>"; // fill in later
static unsigned deltaHeaderCrcOff = 5;
+static unsigned deltaHeaderSizeStart = 21;
+static unsigned deltaHeaderSizeOff = 27;
static unsigned readWriteSlowTracing = 10000; // 10s default
static bool readWriteStackTracing = false;
@@ -870,9 +872,11 @@ void writeDelta(StringBuffer &xml, IFile &iFile, const char *msg="", unsigned re
Owned<IFileIOStream> stream;
offset_t lastGood = 0;
unsigned startCrc = ~0;
- char strNum[11];
+ MemoryBuffer header;
+ char strNum[17];
loop
{
+ header.append(deltaHeader);
try
{
iFileIO.setown(iFile.open(IFOreadwrite));
@@ -897,9 +901,14 @@ void writeDelta(StringBuffer &xml, IFile &iFile, const char *msg="", unsigned re
stream->write(xml.length(), xml.toCharArray());
stream->flush();
stream.clear();
+ offset_t fLen = lastGood + xml.length();
unsigned crc = crc32(xml.toCharArray(), xml.length(), startCrc);
+ char *headerPtr = (char *)header.bufferBase();
sprintf(strNum, "%010u", ~crc);
- iFileIO->write(deltaHeaderCrcOff, 10, strNum);
+ memcpy(headerPtr + deltaHeaderCrcOff, strNum, 10);
+ sprintf(strNum, "%016"I64F"X", fLen);
+ memcpy(headerPtr + deltaHeaderSizeOff, strNum, 16);
+ iFileIO->write(0, strlen(deltaHeader), headerPtr);
}
catch (IException *e)
{
@@ -5137,23 +5146,48 @@ class CStoreHelper : public CInterface, implements IStoreHelper
constructStoreName(DELTADETACHED, deltaInfo.edition, detachName);
return detachName;
}
- virtual bool loadDelta(const char *filename, IFileIO *iFileIO, IPropertyTree *root)
+ virtual bool loadDelta(const char *filename, IFile *iFile, IPropertyTree *root)
{
- OwnedIFileIOStream iFileIOStream = createIOStream(iFileIO);
+ Owned<IFileIO> iFileIO = iFile->open(IFOread);
+ if (!iFileIO) // no delta to load
+ return true;
MemoryBuffer tmp;
char *ptr = (char *) tmp.reserveTruncate(strlen(deltaHeader));
unsigned embeddedCrc = 0;
+ offset_t pos = 0;
bool hasCrcHeader = false; // check really only needed for deltas proceeding CRC header
- if (strlen(deltaHeader) == iFileIOStream->read(strlen(deltaHeader), ptr))
+ if (strlen(deltaHeader) == iFileIO->read(0, strlen(deltaHeader), ptr))
{
if (0 == memicmp(deltaHeader, ptr, 5))
{
+ pos = deltaHeaderSizeStart;
hasCrcHeader = true;
embeddedCrc = (unsigned)atoi64_l(ptr+deltaHeaderCrcOff, 10);
+ if (0 == memicmp(deltaHeader+deltaHeaderSizeStart, ptr+deltaHeaderSizeStart, 6)) // has <SIZE> too
+ {
+ pos = strlen(deltaHeader);
+ offset_t lastGood;
+ if (sscanf(ptr+deltaHeaderSizeOff, "%"I64F"X", &lastGood))
+ {
+ offset_t fSize = iFileIO->size();
+ if (fSize > lastGood)
+ {
+ size32_t diff = fSize - lastGood;
+ LOG(MCoperatorError, unknownJob, "Delta file '%s', has %d bytes of trailing data (possible power loss during save?), file size: %"I64F"d, last committed size: %"I64F"d", filename, diff, fSize, lastGood);
+ LOG(MCoperatorError, unknownJob, "Resetting delta file '%s' to size: %"I64F"d", filename, lastGood);
+ iFileIO->close();
+ backup(filename);
+ iFileIO.setown(iFile->open(IFOreadwrite));
+ iFileIO->setSize(lastGood);
+ iFileIO->close();
+ iFileIO.setown(iFile->open(IFOread));
+ }
+ }
+ }
}
}
- if (!hasCrcHeader)
- iFileIOStream->seek(0, IFSbegin);
+ OwnedIFileIOStream iFileIOStream = createIOStream(iFileIO);
+ iFileIOStream->seek(pos, IFSbegin);
Owned<ICrcIOStream> crcPipeStream = createCrcPipeStream(iFileIOStream); // crc *rest* of stream
Owned<IIOStream> ios = createBufferedIOStream(crcPipeStream);
bool noErrors;
@@ -5184,32 +5218,28 @@ class CStoreHelper : public CInterface, implements IStoreHelper
OwnedIFile deltaIFile = createIFile(deltaFilename.str());
loop
{
- OwnedIFileIO iFileIO;
StringAttr filename;
IFile *iFile;
if (detached)
{
iFile = detachedDeltaIFile;
filename.set(iFile->queryFilename());
- iFileIO.setown(iFile->open(IFOread));
}
else
{
iFile = deltaIFile;
filename.set(iFile->queryFilename());
- iFileIO.setown(iFile->open(IFOread));
- if (!iFileIO) // no delta to load
+ if (!iFile->exists())
break;
}
PROGLOG("Loading delta: %s", filename.get());
bool noError;
Owned<IException> deltaE;
- try { noError = loadDelta(filename, iFileIO, root); }
+ try { noError = loadDelta(filename, iFile, root); }
catch (IException *e) { deltaE.setown(e); noError = false; }
if (!noError)
{
- iFileIO.clear();
backup(filename);
if (errors) *errors = true;
if (deltaE)
View
@@ -233,7 +233,7 @@ extern da_decl unsigned querySDSLockTimeoutCount();
interface IStoreHelper : extends IInterface
{
virtual StringBuffer &getDetachedDeltaName(StringBuffer &detachName) = 0;
- virtual bool loadDelta(const char *filename, IFileIO *iFile, IPropertyTree *root) = 0;
+ virtual bool loadDelta(const char *filename, IFile *iFile, IPropertyTree *root) = 0;
virtual bool loadDeltas(IPropertyTree *root, bool *errors=NULL) = 0;
virtual bool detachCurrentDelta() = 0;
virtual void saveStore(IPropertyTree *root, unsigned *newEdition=NULL, bool currentEdition=false) = 0;
View
@@ -462,7 +462,7 @@ class CDFUengine: public CInterface, implements IDFUengine
void startListener(const char *queuename,CSDSServerStatus *serverstatus)
{
PROGLOG("DFU server waiting on queue %s",queuename);
- cDFUlistener *lt = new cDFUlistener(this,queuename,false,serverstatus, 1000*60);
+ cDFUlistener *lt = new cDFUlistener(this,queuename,false,serverstatus);
listeners.append(*lt);
lt->start();
}
@@ -125,13 +125,11 @@ void coalesceDatastore(bool force)
OwnedIFile detachedIFile = createIFile(detachPath.str());
if (detachedIFile->exists() || iStoreHelper->detachCurrentDelta())
{
- OwnedIFileIO iFileIO = detachedIFile->open(IFOread);
- PROGLOG("COALESCER: Loading delta: %s, size=%"I64F"d", detachName.str(), iFileIO->size());
+ PROGLOG("COALESCER: Loading delta: %s, size=%"I64F"d", detachName.str(), detachedIFile->size());
bool noError;
Owned<IException> deltaE;
- try { noError = iStoreHelper->loadDelta(detachName.str(), iFileIO, root); }
+ try { noError = iStoreHelper->loadDelta(detachName.str(), detachedIFile, root); }
catch (IException *e) { deltaE.setown(e); noError = false; }
- iFileIO.clear();
if (!noError && 0 != (SH_BackupErrorFiles & configFlags))
{
iStoreHelper->backup(detachPath.str());
@@ -147,7 +147,7 @@ int CEclAgentExecutionServer::run()
while (started)
{
PROGLOG("AgentExec: Waiting on queue(s) '%s'", queueNames.str());
- Owned<IJobQueueItem> item = queue->dequeue(WAIT_FOREVER);
+ Owned<IJobQueueItem> item = queue->dequeue();
if (item.get())
{
StringAttr wuid;
@@ -226,12 +226,6 @@ class EclCmdPackageList : public EclCmdCommon
}
virtual bool parseCommandLineOptions(ArgvIterator &iter)
{
- if (iter.done())
- {
- usage();
- return false;
- }
-
for (; !iter.done(); iter.next())
{
const char *arg = iter.query();
@@ -253,12 +247,6 @@ class EclCmdPackageList : public EclCmdCommon
{
if (!EclCmdCommon::finalizeOptions(globals))
return false;
- if (optTarget.isEmpty())
- {
- fprintf(stderr, "\nTarget cluster must be specified\n");
- usage();
- return false;
- }
return true;
}
virtual int processCMD()
@@ -267,6 +255,7 @@ class EclCmdPackageList : public EclCmdCommon
Owned<IClientListPackageRequest> request = packageProcessClient->createListPackageRequest();
request->setTarget(optTarget);
+ request->setProcess("*");
Owned<IClientListPackageResponse> resp = packageProcessClient->ListPackage(request);
if (resp->getExceptions().ordinality())
@@ -350,12 +339,6 @@ class EclCmdPackageInfo: public EclCmdCommon
{
if (!EclCmdCommon::finalizeOptions(globals))
return false;
- if (optTarget.isEmpty())
- {
- fprintf(stderr, "\nTarget cluster must be specified\n");
- usage();
- return false;
- }
return true;
}
virtual int processCMD()
@@ -364,6 +347,7 @@ class EclCmdPackageInfo: public EclCmdCommon
Owned<IClientGetPackageRequest> request = packageProcessClient->createGetPackageRequest();
request->setTarget(optTarget);
+ request->setProcess("*");
Owned<IClientGetPackageResponse> resp = packageProcessClient->GetPackage(request);
if (resp->getExceptions().ordinality())
@@ -433,6 +433,7 @@ class EclccServer : public CInterface, implements IThreadFactory, implements IAb
unsigned maxThreadsActive;
bool running;
CSDSServerStatus serverstatus;
+ Owned<IJobQueue> queue;
public:
IMPLEMENT_IINTERFACE;
@@ -455,15 +456,15 @@ class EclccServer : public CInterface, implements IThreadFactory, implements IAb
void run()
{
DBGLOG("eclccServer (%d threads) waiting for requests on queue(s) %s", poolSize, queueName.get());
- Owned<IJobQueue> queue = createJobQueue(queueName.get());
+ queue.setown(createJobQueue(queueName.get()));
queue->connect();
running = true;
LocalIAbortHandler abortHandler(*this);
while (running)
{
try
{
- Owned<IJobQueueItem> item = queue->dequeue(1000);
+ Owned<IJobQueueItem> item = queue->dequeue();
if (item.get())
{
try
@@ -508,6 +509,8 @@ class EclccServer : public CInterface, implements IThreadFactory, implements IAb
virtual bool onAbort()
{
running = false;
+ if (queue)
+ queue->cancelAcceptConversation();
return false;
}
};
View
@@ -10721,7 +10721,10 @@ void HqlCppTranslator::assignCastUnknownLength(BuildCtx & ctx, const CHqlBoundTa
throwError(HQLERR_CastInfiniteString);
ITranslationInfo * translator = queryDefaultTranslation(tgtset, srcset);
- funcName = createIdentifierAtom(translator->queryVarRtlFunction());
+ if (translator)
+ funcName = createIdentifierAtom(translator->queryVarRtlFunction());
+ else
+ funcName = str2StrXAtom;
}
}
break;
View
@@ -12164,7 +12164,7 @@ void HqlCppTranslator::doBuildAggregateProcessTransform(BuildCtx & ctx, BoundRow
IHqlExpression * src = cur->queryChild(1);
IHqlExpression * arg = src->queryChild(0);
- IHqlExpression * cond = src->queryChild(1);
+ IHqlExpression * cond = queryRealChild(src, 1);
BuildCtx condctx(ctx);
node_operator srcOp = src->getOperator();
View
@@ -0,0 +1,35 @@
+/*##############################################################################
+
+ HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+############################################################################## */
+
+import std;
+Tbl1RecDef := RECORD string10 fname; string10 lname; unsigned1 prange; string10 street; unsigned1 zips; unsigned1 age; string2 birth_state; string3 birth_month; unsigned1 one; unsigned8 id; unsigned8 __filepos {virtual(fileposition)}; END;
+
+Tbl1DS := DATASET('~certification::full_test_distributed', Tbl1RecDef,FLAT);
+
+Idx := INDEX(Tbl1DS, {lname, fname, prange, street, zips, age, birth_state, birth_month},{ __filepos },'~certification::full_test_distributed_index');
+
+IdxDS := Idx(KEYED( age > 10 ) and WILD( lname ) and WILD( fname ) and WILD( prange ) and WILD( street ) and WILD( zips ) and (age > 10 ));
+
+SelectStruct := RECORD
+maxOut := MAX( IdxDS( age > 10 ), IdxDS.age, KEYED );
+string10 lname := IdxDS.lname;
+string10 fname := IdxDS.fname;
+END;
+
+IdxDSTable := TABLE(IdxDS, SelectStruct );
+
+OUTPUT(CHOOSEN(IdxDSTable,100),NAMED('JDBCSelectQueryResult'));
@@ -99,7 +99,7 @@ define([
var paneID = this.getNextPaneID();
var grid = EnhancedGrid({
- result: result,
+ resultIndex: resultIndex,
store: result.getObjectStore(),
query: { id: "*" },
structure: result.getStructure(),
@@ -72,6 +72,7 @@ ESPresponse [exceptions_inline] DeActivatePackageResponse
ESPrequest GetPackageRequest
{
string Target;
+ string Process;
};
ESPresponse [exceptions_inline] GetPackageResponse
@@ -83,6 +84,7 @@ ESPresponse [exceptions_inline] GetPackageResponse
ESPrequest ListPackageRequest
{
string Target;
+ string Process;
};
ESPstruct PackageListData
@@ -94,6 +96,7 @@ ESPstruct PackageListData
ESPstruct PackageListMapData
{
string Id;
+ string Target;
ESParray<ESPstruct PackageListData> PkgListData;
};
Oops, something went wrong.

0 comments on commit fcedcca

Please sign in to comment.