Skip to content

Commit

Permalink
HPCC-16870 Add method to rebuild a query from its ECL archive
Browse files Browse the repository at this point in the history 
This will be really useful when upgrading the ECL compiler, allowing
the user to rebuild their query from the exact same code.

Signed-off-by: Anthony Fishbeck <anthony.fishbeck@lexisnexis.com>
  • Loading branch information
@afishbeck
afishbeck committed Sep 21, 2017
1 parent dcb0d7d commit 3f2fe5c
Show file tree
Hide file tree
Showing 8 changed files with 548 additions and 17 deletions.
1 change: 1 addition & 0 deletions ecl/eclcmd/eclcmd_common.hpp
View file
Expand Up @@ -67,6 +67,7 @@ typedef IEclCommand *(*EclCommandFactory)(const char *cmdname);
#define ECLOPT_PASSWORD_ENV "ECL_PASSWORD"

#define ECLOPT_NORELOAD "--no-reload"
#define ECLOPT_NOPUBLISH "--no-publish"
#define ECLOPT_OVERWRITE "--overwrite"
#define ECLOPT_REPLACE "--replace"
#define ECLOPT_OVERWRITE_S "-O"
Expand Down
270 changes: 270 additions & 0 deletions ecl/eclcmd/queries/ecl-queries.cpp
View file
Expand Up @@ -910,6 +910,273 @@ class EclCmdQueriesConfig : public EclCmdCommon
bool optNoReload;
};

class EclCmdQueriesRecreate : public EclCmdCommon
{
public:
EclCmdQueriesRecreate()
{
}
virtual eclCmdOptionMatchIndicator parseCommandLineOptions(ArgvIterator &iter)
{
if (iter.done())
return EclCmdOptionNoMatch;

for (; !iter.done(); iter.next())
{
const char *arg = iter.query();
if (*arg!='-')
{
if (optTarget.isEmpty())
optTarget.set(arg);
else if (optQueryId.isEmpty())
optQueryId.set(arg);
else if (optDestTarget.isEmpty())
optDestTarget.set(arg);
else
{
fprintf(stderr, "\nunrecognized argument %s\n", arg);
return EclCmdOptionNoMatch;
}
continue;
}
if (iter.matchOption(optDaliIP, ECLOPT_DALIIP))
continue;
if (iter.matchOption(optSourceProcess, ECLOPT_SOURCE_PROCESS))
continue;
if (iter.matchOption(optMsToWait, ECLOPT_WAIT))
continue;
if (iter.matchOption(optTimeLimit, ECLOPT_TIME_LIMIT))
continue;
if (iter.matchOption(optWarnTimeLimit, ECLOPT_WARN_TIME_LIMIT))
continue;
if (iter.matchOption(optMemoryLimit, ECLOPT_MEMORY_LIMIT))
continue;
if (iter.matchOption(optPriority, ECLOPT_PRIORITY))
continue;
if (iter.matchOption(optComment, ECLOPT_COMMENT))
continue;
if (iter.matchFlag(optDontCopyFiles, ECLOPT_DONT_COPY_FILES))
continue;
if (iter.matchFlag(optAllowForeign, ECLOPT_ALLOW_FOREIGN))
continue;
if (iter.matchFlag(optNoActivate, ECLOPT_NO_ACTIVATE))
{
activateSet=true;
continue;
}
if (iter.matchFlag(optNoReload, ECLOPT_NORELOAD))
continue;
if (iter.matchFlag(optNoPublish, ECLOPT_NOPUBLISH))
continue;
bool activate; //also supports "-A-"
if (iter.matchFlag(activate, ECLOPT_ACTIVATE)||iter.matchFlag(activate, ECLOPT_ACTIVATE_S))
{
activateSet=true;
optNoActivate=!activate;
continue;
}
if (iter.matchFlag(optSuspendPrevious, ECLOPT_SUSPEND_PREVIOUS)||iter.matchFlag(optSuspendPrevious, ECLOPT_SUSPEND_PREVIOUS_S))
continue;
if (iter.matchFlag(optDeletePrevious, ECLOPT_DELETE_PREVIOUS)||iter.matchFlag(optDeletePrevious, ECLOPT_DELETE_PREVIOUS_S))
continue;
if (iter.matchFlag(optUpdateDfs, ECLOPT_UPDATE_DFS))
continue;
if (iter.matchFlag(optUpdateSuperfiles, ECLOPT_UPDATE_SUPER_FILES))
continue;
if (iter.matchFlag(optUpdateCloneFrom, ECLOPT_UPDATE_CLONE_FROM))
continue;
if (iter.matchFlag(optDontAppendCluster, ECLOPT_DONT_APPEND_CLUSTER))
continue;
if (iter.matchOption(optResultLimit, ECLOPT_RESULT_LIMIT))
continue;
if (matchVariableOption(iter, 'f', debugValues))
continue;
eclCmdOptionMatchIndicator ind = EclCmdCommon::matchCommandLineOption(iter, true);
if (ind != EclCmdOptionMatch)
return ind;
}
return EclCmdOptionMatch;
}
virtual bool finalizeOptions(IProperties *globals)
{
if (!EclCmdCommon::finalizeOptions(globals))
return false;
if (optTarget.isEmpty())
{
fputs("Target not specified.\n", stderr);
return false;
}
if (optQueryId.isEmpty())
{
fputs("Query not specified.\n", stderr);
return false;
}
if (!activateSet)
{
bool activate;
if (extractEclCmdOption(activate, globals, ECLOPT_ACTIVATE_ENV, ECLOPT_ACTIVATE_INI, true))
optNoActivate=!activate;
}
if (optNoActivate && (optSuspendPrevious || optDeletePrevious))
{
fputs("invalid --suspend-prev and --delete-prev require activation.\n", stderr);
return false;
}
if (!optSuspendPrevious && !optDeletePrevious)
{
extractEclCmdOption(optDeletePrevious, globals, ECLOPT_DELETE_PREVIOUS_ENV, ECLOPT_DELETE_PREVIOUS_INI, false);
if (!optDeletePrevious)
extractEclCmdOption(optSuspendPrevious, globals, ECLOPT_SUSPEND_PREVIOUS_ENV, ECLOPT_SUSPEND_PREVIOUS_INI, false);
}
if (optSuspendPrevious && optDeletePrevious)
{
fputs("invalid --suspend-prev and --delete-prev are mutually exclusive options.\n", stderr);
return false;
}
if (optMemoryLimit.length() && !isValidMemoryValue(optMemoryLimit))
{
fprintf(stderr, "invalid --memoryLimit value of %s.\n", optMemoryLimit.get());
return false;
}
if (optPriority.length() && !isValidPriorityValue(optPriority))
{
fprintf(stderr, "invalid --priority value of %s.\n", optPriority.get());
return false;
}
return true;
}
virtual int processCMD()
{
if (optVerbose)
fprintf(stdout, "\nRecreating %s/%s\n", optTarget.str(), optQueryId.str());

Owned<IClientWsWorkunits> client = createCmdClient(WsWorkunits, *this); //upload_ disables maxRequestEntityLength
Owned<IClientWURecreateQueryRequest> req = client->createWURecreateQueryRequest();

if (optDeletePrevious)
req->setActivate(CWUQueryActivationMode_ActivateDeletePrevious);
else if (optSuspendPrevious)
req->setActivate(CWUQueryActivationMode_ActivateSuspendPrevious);
else
req->setActivate(optNoActivate ? CWUQueryActivationMode_NoActivate : CWUQueryActivationMode_Activate);

req->setTarget(optTarget);
req->setDestTarget(optDestTarget);
req->setQueryId(optQueryId);
req->setRemoteDali(optDaliIP);
req->setSourceProcess(optSourceProcess);
req->setWait(optMsToWait);
req->setNoReload(optNoReload);
req->setRepublish(!optNoPublish);
req->setDontCopyFiles(optDontCopyFiles);
req->setAllowForeignFiles(optAllowForeign);
req->setUpdateDfs(optUpdateDfs);
req->setUpdateSuperFiles(optUpdateSuperfiles);
req->setUpdateCloneFrom(optUpdateCloneFrom);
req->setAppendCluster(!optDontAppendCluster);
req->setIncludeFileErrors(true);
req->setDebugValues(debugValues);

if (optTimeLimit != (unsigned) -1)
req->setTimeLimit(optTimeLimit);
if (optWarnTimeLimit != (unsigned) -1)
req->setWarnTimeLimit(optWarnTimeLimit);
if (!optMemoryLimit.isEmpty())
req->setMemoryLimit(optMemoryLimit);
if (!optPriority.isEmpty())
req->setPriority(optPriority);
if (optComment.get()) //allow empty
req->setComment(optComment);

Owned<IClientWURecreateQueryResponse> resp = client->WURecreateQuery(req);
const char *wuid = resp->getWuid();
if (wuid && *wuid)
fprintf(stdout, "\nWorkunit: %s\n", wuid);
const char *id = resp->getQueryId();
if (id && *id)
{
const char *qs = resp->getQuerySet();
fprintf(stdout, "\nPublished: %s/%s\n", qs ? qs : "", resp->getQueryId());
}
if (resp->getReloadFailed())
fputs("\nAdded to Target, but request to reload queries on cluster failed\n", stderr);

int ret = outputMultiExceptionsEx(resp->getExceptions());
if (outputQueryFileCopyErrors(resp->getFileErrors()))
ret = 1;

return ret;
}
virtual void usage()
{
fputs("\nUsage:\n"
"\n"
"The 'queries recreate' command recompiles a query into a new workunit and republishes\n"
"the new workunit. This is usefull when upgrading to a new ECL compiler and you\n"
"want to recompile a query with the exact same source.\n"
"\n"
"The ECL archive must be available within the workunit of the query.\n"
"\n"
"ecl queries recreate <target> <query> [options]\n\n"
"ecl queries recreate <target> <query> <destination-target> [options]\n\n"
" <target> the target the query you wish to recreate is in\n"
" <query> the query ID of the query you wish to recreate\n"
" <destination-target> the target you want to move the new query to\n"
" (if different from the source target\n"
" Options:\n"
" -A, --activate Activate query when published (default)\n"
" --limit=<limit> Sets the result limit for the query, defaults to 100\n"
" -sp, --suspend-prev Suspend previously active query\n"
" -dp, --delete-prev Delete previously active query\n"
" -A-, --no-activate Do not activate query when published\n"
" --no-publish Create a recompiled workunit, but do not publish it\n"
" --no-reload Do not request a reload of the (roxie) cluster\n"
" --no-files Do not copy DFS file information for referenced files\n"
" --allow-foreign Do not fail if foreign files are used in query (roxie)\n"
" --daliip=<IP> The IP of the DALI to be used to locate remote files\n"
" --update-super-files Update local DFS super-files if remote DALI has changed\n"
" --update-clone-from Update local clone from location if remote DALI has changed\n"
" --dont-append-cluster Only use to avoid locking issues due to adding cluster to file\n"
" --source-process Process cluster to copy files from\n"
" --timeLimit=<ms> Value to set for query timeLimit configuration\n"
" --warnTimeLimit=<ms> Value to set for query warnTimeLimit configuration\n"
" --memoryLimit=<mem> Value to set for query memoryLimit configuration\n"
" format <mem> as 500000B, 550K, 100M, 10G, 1T etc.\n"
" --priority=<val> set the priority for this query. Value can be LOW,\n"
" HIGH, SLA, NONE. NONE will clear current setting.\n"
" --comment=<string> Set the comment associated with this query\n"
" --wait=<ms> Max time to wait in milliseconds\n",
stdout);
EclCmdCommon::usage();
}
private:
StringAttr optTarget;
StringAttr optDestTarget;
StringAttr optQueryId;
StringAttr optDaliIP;
StringAttr optSourceProcess;
StringAttr optMemoryLimit;
StringAttr optPriority;
StringAttr optComment;
IArrayOf<IEspNamedValue> debugValues;
unsigned optMsToWait = (unsigned) -1;
unsigned optTimeLimit = (unsigned) -1;
unsigned optWarnTimeLimit = (unsigned) -1;
unsigned optResultLimit = (unsigned) -1;
bool optNoActivate = false;
bool activateSet = false;
bool optNoReload = false;
bool optNoPublish = false;
bool optDontCopyFiles = false;
bool optSuspendPrevious = false;
bool optDeletePrevious = false;
bool optAllowForeign = false;
bool optUpdateDfs = false;
bool optUpdateSuperfiles = false;
bool optUpdateCloneFrom = false;
bool optDontAppendCluster = false; //Undesirable but here temporarily because DALI may have locking issues
};

IEclCommand *createEclQueriesCommand(const char *cmdname)
{
if (!cmdname || !*cmdname)
Expand All @@ -924,6 +1191,8 @@ IEclCommand *createEclQueriesCommand(const char *cmdname)
return new EclCmdQueriesCopy();
if (strieq(cmdname, "copy-set"))
return new EclCmdQueriesCopyQueryset();
if (strieq(cmdname, "recreate"))
return new EclCmdQueriesRecreate();
return NULL;
}

Expand All @@ -947,6 +1216,7 @@ class EclQueriesCMDShell : public EclCMDShell
" config update query settings\n"
" copy copy a query from one target cluster to another\n"
" copy-set copy queries from one target cluster to another\n"
" recreate recompiles query into a new workunit\n"
);
}
};
Expand Down
67 changes: 58 additions & 9 deletions esp/scm/ws_workunits.ecm
View file
Expand Up @@ -597,6 +597,62 @@ ESPresponse [exceptions_inline] WUResubmitResponse
[min_ver("1.40")] ESParray<ESPstruct ResubmittedWU, WU> WUs;
};

ESPenum WUQueryActivationMode : int
{
NoActivate(0, "Do not activate query"),
Activate(1, "Activate query"),
ActivateSuspendPrevious(2, "Activate query, suspend previous"),
ActivateDeletePrevious(3, "Activate query, delete previous")
};

ESPrequest [nil_remove] WURecreateQueryRequest
{
string Target;
string QueryId;
ESParray<ESPstruct NamedValue> DebugValues;
string DestTarget;
bool Republish(0);
ESPEnum WUQueryActivationMode Activate;
bool NoReload(0);

string MemoryLimit;
nonNegativeInteger TimeLimit(0);
nonNegativeInteger WarnTimeLimit(0);
string Priority;
string Comment;

string RemoteDali;
bool DontCopyFiles(false);
string SourceProcess;
bool AllowForeignFiles(false);
bool UpdateDfs(false);
bool UpdateSuperFiles(false); //update content of superfiles if changed
bool UpdateCloneFrom(false); //explicity wan't to change where roxie will grab from
bool AppendCluster(true); //file exists on other local cluster, add new one, make optional in case of locking issues, but should be made to work
bool IncludeFileErrors(false);

int Wait(-1);
};

ESPresponse [exceptions_inline, nil_remove] WURecreateQueryResponse
{
string Wuid;
string QuerySet;
string QueryName;
string QueryId;

string MemoryLimit;
nonNegativeInteger TimeLimit;
nonNegativeInteger WarnTimeLimit;
string Priority;
string Comment;

bool ReloadFailed;
bool Suspended;
string ErrorMessage;
ESParray<ESPStruct LogicalFileError, File> FileErrors;
};

ESPenum WUExceptionSeverity : string
{
INFO("info"),
Expand Down Expand Up @@ -1221,14 +1277,6 @@ ESPresponse [exceptions_inline] WUCopyLogicalFilesResponse
};


ESPenum WUQueryActivationMode : int
{
NoActivate(0, "Do not activate query"),
Activate(1, "Activate query"),
ActivateSuspendPrevious(2, "Activate query, suspend previous"),
ActivateDeletePrevious(3, "Activate query, delete previous")
};

ESPrequest [nil_remove] WUPublishWorkunitRequest
{
string Wuid;
Expand Down Expand Up @@ -1828,7 +1876,7 @@ ESPresponse [exceptions_inline, nil_remove] WUGetNumFileToCopyResponse

ESPservice [
auth_feature("DEFERRED"), //This declares that the method logic handles feature level authorization
version("1.69"), default_client_version("1.69"),
version("1.70"), default_client_version("1.70"),
noforms,exceptions_inline("./smc_xslt/exceptions.xslt"),use_method_name] WsWorkunits
{
ESPmethod [cache_seconds(60), resp_xsl_default("/esp/xslt/workunits.xslt")] WUQuery(WUQueryRequest, WUQueryResponse);
Expand Down Expand Up @@ -1864,6 +1912,7 @@ ESPservice [

ESPmethod WUAbort(WUAbortRequest, WUAbortResponse);
ESPmethod WUProtect(WUProtectRequest, WUProtectResponse);
ESPmethod [min_ver("1.70")] WURecreateQuery(WURecreateQueryRequest, WURecreateQueryResponse);
ESPmethod WUResubmit(WUResubmitRequest, WUResubmitResponse); //????
ESPmethod WURun(WURunRequest, WURunResponse);

Expand Down

0 comments on commit 3f2fe5c

Please sign in to comment.