HPCC-14396 REFRESH(boolean) option on persist

 Please enter the commit message for your changes. Lines starting

ecl/eclagent/eclagent.cpp

@@ -2498,12 +2498,12 @@ bool EclAgent::checkPersistUptoDate(IRuntimeWorkflowItem & item, const char * lo
 
     if (!isResult(lfn, (unsigned)-2))
         errText.appendf("Building PERSIST('%s'): It hasn't been calculated before", logicalName);
-    else if (!isResult(crcName, (unsigned)-2))
-        errText.appendf("Rebuilding PERSIST('%s'): Saved CRC isn't present", logicalName);
     else if (isFile && !fileExists(logicalName))
         errText.appendf("Rebuilding PERSIST('%s'): Persistent file does not exist", logicalName);
     else if (!item.queryPersistRefresh())
         return true;
+    else if (!isResult(crcName, (unsigned)-2))
+        errText.appendf("Rebuilding PERSIST('%s'): Saved CRC isn't present", logicalName);
     else
     {
         unsigned savedEclCRC = (unsigned)getResultInt(eclName, (unsigned)-2);

ecl/hqlcpp/hqlttcpp.cpp

@@ -5080,10 +5080,7 @@ void GlobalAttributeInfo::extractStoredInfo(IHqlExpression * expr, IHqlExpressio
             getStringValue(s, codehash);
             storedName.setown(createConstant(s.str()));
         }
-        if (expr->hasAttribute(refreshAtom))
-            persistRefresh = getBoolValue(queryAttributeChild(expr,refreshAtom,0), true);
-        else
-            persistRefresh = true;
+	persistRefresh = getBoolValue(queryAttributeChild(expr, refreshAtom, 0), true);
         break;
     case no_global:
         throwUnexpected();

roxie/ccd/ccdcontext.cpp

@@ -360,10 +360,12 @@ class CRoxieWorkflowMachine : public WorkflowMachine
 
         if (!isResult(lfn, ResultSequencePersist))
             errText.appendf("Building PERSIST('%s'): It hasn't been calculated before", logicalName);
-        else if (!isResult(crcName, ResultSequencePersist))
-            errText.appendf("Rebuilding PERSIST('%s'): Saved CRC isn't present", logicalName);
         else if (isFile && !fileExists(lfn))
             errText.appendf("Rebuilding PERSIST('%s'): Persistent file does not exist", logicalName);
+	else if (!item.queryPersistRefresh())
+	  return true;
+        else if (!isResult(crcName, ResultSequencePersist))
+            errText.appendf("Rebuilding PERSIST('%s'): Saved CRC isn't present", logicalName);
         else
         {
             unsigned savedEclCRC = (unsigned) getResultInt(eclName, ResultSequencePersist);

testing/regress/download/europe1.txt

@@ -0,0 +1,5 @@
+#Country,Population
+Spain ,40397842
+Sweden ,9016596
+Switzerland ,7523934
+United Kingdom ,60609153

testing/regress/download/europe2.txt

@@ -0,0 +1,5 @@
+#Country,Population,Area,Pop. Density,Literacy,Phones
+Spain ,40397842,504782,80.0,97.9,453.5
+Sweden ,9016596,449964,20.0,99.0,715.0
+Switzerland ,7523934,41290,182.2,99.0,680.9
+UK ,60609153,244820,247.6,99.0,543.5

testing/regress/ecl/key/persist_refresh.xml

@@ -0,0 +1,6 @@
+<Dataset name='Result 1'>
+ <Row><country>Spain</country><population>40397842</population></Row>
+ <Row><country>Sweden</country><population>9016596</population></Row>
+ <Row><country>Switzerland</country><population>7523934</population></Row>
+ <Row><country>United Kingdom</country><population>60609153</population></Row>
+</Dataset>

testing/regress/ecl/persist_refresh.ecl

@@ -0,0 +1,72 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+import Std;
+import Std.File AS FileServices;
+
+//
+//version persistRefresh=true
+//version persistRefresh=false
+
+import ^ as root;
+
+string OriginalTextFilesIp := '.' : STORED('OriginalTextFilesIp');
+string OriginalTextFilesEclPath := '' : STORED('OriginalTextFilesEclPath');
+string OriginalTextFilesOsPath := '' : STORED('OriginalTextFilesOsPath');
+
+persistRefresh := #IFDEFINED(root.persistRefresh, true);
+deleteFiles := #IFDEFINED(root.deleteFiles, true);
+
+#IF (persistRefresh=true)
+InputFile := OriginalTextFilesOsPath + '/download/europe1.txt';
+#ELSE
+InputFile := OriginalTextFilesOsPath + '/download/europe2.txt';
+#END
+
+ClusterName := 'mythor';
+DestFile := '~regress::europe.txt';
+ESPportIP := 'http://127.0.0.1:8010/FileSpray';
+
+spray := FileServices.SprayVariable(OriginalTextFilesIp,
+                                     InputFile,
+                                     destinationGroup := ClusterName,
+                                     destinationLogicalName := DestFile,
+                                     espServerIpPort := ESPportIP,
+                                     ALLOWOVERWRITE := true);
+
+rec := RECORD
+    string country;
+    integer4 population;
+END;
+
+ds := DATASET(DestFile, rec, CSV(HEADING(1), SEPARATOR(','), QUOTE([])));
+
+#IF (persistRefresh=true)
+    CountriesDS := ds:PERSIST('~REGRESS::PersistRefresh', SINGLE, REFRESH(true));
+#ELSE
+    CountriesDS := ds:PERSIST('~REGRESS::PersistRefresh', SINGLE, REFRESH(false));
+#END
+
+OutputResults := OUTPUT(CountriesDS);
+
+pause := STD.System.Debug.Sleep(2000);
+
+sequential (
+       spray;
+       pause; 
+       OutputResults;
+) ;