Skip to content

Commit

Permalink
Added in support for validation run
Browse files Browse the repository at this point in the history 
By setting a flag in the config.json, carries out a validation run using
a pre-defined list of streets which incorporate all items. In addition
JSONDiff output is put into the validation output file. This validation
run also uses as source JSON files in a special directory in the
validation directory. Therefore in the future the tool can be re-run to
check the output is the same - thereby checking that changes have not
broken anything.
  • Loading branch information
@jarkW
jarkW committed Jun 10, 2016
1 parent 1944a17 commit 2baafa7
Show file tree
Hide file tree
Showing 5 changed files with 209 additions and 14 deletions.
79 changes: 71 additions & 8 deletions ConfigInfo.pde
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ class ConfigInfo {

boolean debugShowBWFragments;
boolean debugDumpDiffImages;
boolean debugValidationRun;

StringList streetTSIDs = new StringList();
String outputFile;
Expand Down Expand Up @@ -68,6 +69,13 @@ class ConfigInfo {
return false;
}

// Do this first - as might need to reset the vagrant flag
debugValidationRun = Utils.readJSONBool(json, "debug_validation_run", false);
if (!Utils.readOkFlag())
{
debugValidationRun = false;
}

// Now read in the different fields
useVagrant = Utils.readJSONBool(json, "use_vagrant_dirs", true);
if (!Utils.readOkFlag())
Expand All @@ -78,6 +86,12 @@ class ConfigInfo {
return false;
}

// If running a validation run, then force it to use vagrant files
if (debugValidationRun)
{
useVagrant = true;
}

// Read in the locations of the JSON directories
JSONObject fileSystemInfo;
if (useVagrant)
Expand Down Expand Up @@ -285,8 +299,7 @@ class ConfigInfo {
return false;
}




// The following options are OPTIONAL - so don't need to be in the JSON file
changeXYOnly = Utils.readJSONBool(json, "change_xy_only", false);
if (!Utils.readOkFlag())
Expand Down Expand Up @@ -330,29 +343,74 @@ class ConfigInfo {
return false;
}

// THESE ARE ONLY USED FOR DEBUG TESTING - so not error if missing
debugShowBWFragments = Utils.readJSONBool(json, "debug_show_BW_fragments", false);
if (!Utils.readOkFlag())
{
debugShowBWFragments = false;
}

debugDumpDiffImages = Utils.readJSONBool(json, "debug_dump_diff_images", false);
if (!Utils.readOkFlag())
{
debugDumpDiffImages = false;
}

if (!usingBlackWhiteComparison && debugDumpDiffImages)
{
println("usingBlackWhiteComparison is set to false, so debug_dump_bw_diff_images in config.json cannot be set to true");
displayMgr.showErrMsg("usingBlackWhiteComparison is set to false, so debug_dump_bw_diff_images in config.json cannot be set to true", true);
return false;
}

// THESE ARE ONLY USED FOR DEBUG TESTING - so not error if missing
debugShowBWFragments = Utils.readJSONBool(json, "debug_show_BW_fragments", false);
if (!Utils.readOkFlag())
// Default different fields so that validation runs always do the same testing
if (debugValidationRun)
{
debugShowBWFragments = false;
debugLevel = 1;
percentMatchCriteria = 90;
searchRadius = 25;
changeXYOnly = false;
writeJSONsToPersdata = false;

//Reset paths to snaps and persdata (so always use the same set of original JSON files)

String validationPath = Utils.readJSONString(json, "debug_validation_path", true);
if (!Utils.readOkFlag() || validationPath.length() == 0)
{
println(Utils.readErrMsg());
println("Failed to read debug_validation_path in config.json file");
displayMgr.showErrMsg("Failed to read debug_validation_path in config.json file", true);
return false;
}
outputFile = validationPath + File.separatorChar + "validation.txt";
streetSnapPath = validationPath + File.separatorChar + "Snaps";
myDir = new File(streetSnapPath);
if (!myDir.exists())
{
println("Validation street snap archive directory ", streetSnapPath, " does not exist");
displayMgr.showErrMsg("Validation street snap archive directory " + streetSnapPath + " does not exist", true);
return false;
}
persdataPath = validationPath + File.separatorChar + "JSONs";
myDir = new File(persdataPath);
if (!myDir.exists())
{
println("Validation source JSON directory ", persdataPath, " does not exist");
displayMgr.showErrMsg("Validation source JSON directory " + persdataPath + " does not exist", true);
return false;
}
}
// End of debug only info

// Read in array of street TSID from config file
JSONArray TSIDArray = Utils.readJSONArray(json, "streets", true);
JSONArray TSIDArray;
if (debugValidationRun)
{
TSIDArray = Utils.readJSONArray(json, "streets_validation", true);
}
else
{
TSIDArray = Utils.readJSONArray(json, "streets", true);
}
if (!Utils.readOkFlag())
{
println(Utils.readErrMsg());
Expand Down Expand Up @@ -505,4 +563,9 @@ class ConfigInfo {
return debugDumpDiffImages;
}

public boolean readDebugValidationRun()
{
return debugValidationRun;
}

}
13 changes: 13 additions & 0 deletions ItemInfo.pde
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ class ItemInfo
// Only used for debug purposes - to collect y values on the street with reference quoins on
IntList itemYValues;
boolean collectItemYValues = false;
String validationInfo;

// constructor/initialise fields
public ItemInfo(JSONObject item)
Expand All @@ -60,6 +61,7 @@ class ItemInfo
saveChangedJSONfile = false;

itemYValues = new IntList();
validationInfo = "";

bestMatchInfoList = new ArrayList<MatchInfo>();

Expand Down Expand Up @@ -964,6 +966,12 @@ class ItemInfo
}
// Displays message to user in both debug files
jsonDiff.displayInfoMsg(false);

// Save the JSON diff information validation runs - will be later printed out in the output file so easy to view
if (configInfo.readDebugValidationRun())
{
validationInfo = jsonDiff.readValidationInfo();
}

} // end if changes to save to JSON file
else
Expand Down Expand Up @@ -1344,6 +1352,11 @@ class ItemInfo
return okFlag;
}

public String readValidationInfo()
{
return validationInfo;
}

public MatchInfo readBestMatchInfo()
{
if (bestMatchInfoList.size() < 1)
Expand Down
111 changes: 105 additions & 6 deletions JSONDiff.pde
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ class JSONDiff
// Using list of information strings - so that if nothing found, just get simple one line message
// User has to request the information be printed
StringList infoMsgList;
String validationInfo;
String itemTSID;
String itemClassTSID;
JSONObject origJSON;
Expand All @@ -23,6 +24,7 @@ class JSONDiff
public JSONDiff(String iTSID, String iClassTSID, String origJSONFileName, String newJSONFileName)
{
infoMsgList = new StringList();
validationInfo = "";
itemTSID = iTSID;
itemClassTSID = iClassTSID;
origJSON = null;
Expand All @@ -43,6 +45,10 @@ class JSONDiff

s = "JSONDIFF: Starting compare of item file " + itemTSID + ".json";
infoMsgList.append(s);
if (configInfo.readDebugValidationRun())
{
validationInfo = "JSONDIFF: " + itemTSID + ": ";
}

if (!compareJSONObjects(origJSON, newJSON, ""))
{
Expand Down Expand Up @@ -128,7 +134,31 @@ class JSONDiff
// Search for matching keys in file - trying to find a match in origList for the new key
while (!newList.get(i).equals(origList.get(j)) && (i < newList.size()))
{
s = "New key in new JSON file is " + newList.get(i);
if (!extractObjectFromJSONObject(newVersion, newList, i))
{
// Unexpected type of object in JSON
s = "ERROR - unexpected type of object in JSON file" + newFileName;
infoMsgList.append(s);
return false;
}
// extractedObject is set by extractObjectFromJSONObject ()
if (!(extractedObject == null) && !(extractedObject instanceof JSONObject) && !(extractedObject instanceof JSONArray))
{
// Means we have a bool/int/String we can dump out
s = "New key in new JSON file is " + newList.get(i) + " set to " + extractedObject;
if (configInfo.readDebugValidationRun())
{
validationInfo = validationInfo + newList.get(i) + " (new->" + extractedObject + "); ";
}
}
else
{
s = "New key in new JSON file is " + newList.get(i) + " set to ????????";
if (configInfo.readDebugValidationRun())
{
validationInfo = validationInfo + newList.get(i) + " (new->????); ";
}
}
infoMsgList.append(s);
i++;
}
Expand All @@ -154,8 +184,32 @@ class JSONDiff
return false;
}
else
{
s = "Valid new key (" + newList.get(j) + ") at end of new file";
{
if (!extractObjectFromJSONObject(newVersion, newList, j))
{
// Unexpected type of object in JSON
s = "ERROR - unexpected type of object in JSON file" + newFileName;
infoMsgList.append(s);
return false;
}
// extractedObject is set by extractObjectFromJSONObject ()
if (!(extractedObject == null) && !(extractedObject instanceof JSONObject) && !(extractedObject instanceof JSONArray))
{
// Means we have a bool/int/String we can dump out
s = "Valid new key at end of file - " + newList.get(j) + " set to " + extractedObject;
if (configInfo.readDebugValidationRun())
{
validationInfo = validationInfo + newList.get(j) + " (new (end file)->" + extractedObject + "); ";
}
}
else
{
s = "Valid new key at end of file - " + newList.get(j) + " set to ????????";
if (configInfo.readDebugValidationRun())
{
validationInfo = validationInfo + newList.get(j) + " (new (end file)->????); ";
}
}
infoMsgList.append(s);
return true;
}
Expand Down Expand Up @@ -197,9 +251,25 @@ class JSONDiff
else if (origObj == null)
{
// Could this be the case when field not set, and we then set it correctly using the tool
s = "WARNING " + itemTSID + ".json, at level " + level + "/" + newList.get(i) + " New key is non-null, orig key is null";
if (!(newObj instanceof JSONObject) && !(newObj instanceof JSONArray))
{
// Means we have a bool/int/String we can dump out
s = "WARNING " + itemTSID + ".json, at level " + level + "/" + newList.get(i) + " new key is set to " + newObj + " orig key is null";
if (configInfo.readDebugValidationRun())
{
validationInfo = validationInfo + newList.get(i) + " (null->" + newObj + "); ";
}
}
else
{
s = "WARNING " + itemTSID + ".json, at level " + level + "/" + newList.get(i) + " new key is set to ??????? orig key is null";
if (configInfo.readDebugValidationRun())
{
validationInfo = validationInfo + newList.get(i) + " (null->????); ";
}
}
infoMsgList.append(s);
}
}
else if (newObj instanceof JSONObject && origObj instanceof JSONObject)
{
if (!compareJSONObjects((JSONObject)origObj, (JSONObject)newObj, level + "/" + newList.get(i)))
Expand Down Expand Up @@ -248,6 +318,10 @@ class JSONDiff
{
s = "Different values found for valid " + level + "/" + newList.get(i) + " new value is <" + newObj + "> was <" + origObj + ">";
infoMsgList.append(s);
if (configInfo.readDebugValidationRun())
{
validationInfo = validationInfo + newList.get(i) + "(" + origObj + "->" + newObj + "); ";
}
}
}
else
Expand Down Expand Up @@ -312,7 +386,23 @@ class JSONDiff
else if (origObj == null)
{
// This could be valid if the tool sets up a field which was originally null
s = "WARNING in " + itemTSID + ".json, at level " + level + "/" + i + "New index i is non-null orig index i is -null";
if (!(newObj instanceof JSONObject) && !(newObj instanceof JSONArray))
{
// Means we have a bool/int/String we can dump out
s = "WARNING " + itemTSID + ".json, at level " + level + "/" + i + " new value is " + newObj + " orig value is null";
if (configInfo.readDebugValidationRun())
{
validationInfo = validationInfo + level + "/" + i + " (WARNING null->" + newObj + "); ";
}
}
else
{
s = "WARNING " + itemTSID + ".json, at level " + level + "/" + i + " new value is ??????? orig value is null";
if (configInfo.readDebugValidationRun())
{
validationInfo = validationInfo + level + "/" + i + " (WARNING null->???????); ";
}
}
infoMsgList.append(s);
}
else if (newObj instanceof JSONArray && origObj instanceof JSONArray)
Expand Down Expand Up @@ -348,6 +438,10 @@ class JSONDiff
// Difference found in values in array
s = "Different values found for " + level + "/" + i + " new value is <" + newObj + "> was <" + origObj + ">";
infoMsgList.append(s);
if (configInfo.readDebugValidationRun())
{
validationInfo = validationInfo + level + "/" + i + "(" + origObj + "->" + newObj + "); ";
}
}
else
{
Expand Down Expand Up @@ -686,4 +780,9 @@ class JSONDiff

}

public String readValidationInfo()
{
return validationInfo;
}

}
6 changes: 6 additions & 0 deletions PrintToFile.pde
View file
Original file line number Diff line number Diff line change
Expand Up @@ -507,6 +507,12 @@ class PrintToFile {
}

printOutputLine(s);

// Now print out the JSON Diff info if it exists
if (configInfo.readDebugValidationRun() && itemResults.get(i).itemInfo.readValidationInfo().length() > 0)
{
printOutputLine(itemResults.get(i).itemInfo.readValidationInfo());
}

if (itemResults.get(i).itemInfo.readItemClassTSID().equals("quoin"))
{
Expand Down
14 changes: 14 additions & 0 deletions StreetInfo.pde
View file
Original file line number Diff line number Diff line change
Expand Up @@ -717,6 +717,14 @@ class StreetInfo
boolean streetNotExistInPersdataQA(String streetTSID)
{

if (configInfo.readDebugValidationRun())
{
// For validation runs we never write to persdata, so doesn't matter if the
// street already exists in persdata-qa - irrelevant
printToFile.printDebugLine(this, "Skip persdata-qa test for street TSID " + streetTSID, 1);
return true;
}

if (configInfo.readUseVagrantFlag())
{
File myDir = new File(configInfo.readPersdataQAPath() + File.separatorChar + streetTSID);
Expand Down Expand Up @@ -1037,6 +1045,12 @@ class StreetInfo
File file = new File(sourcePath);
if (!file.exists())
{
if (configInfo.readDebugValidationRun())
{
// Should never reach this point - file should have been read from the special validation JSON directory
printToFile.printDebugLine(this, "Unable to find validation JSON file on vagrant - " + sourcePath, 3);
return false;
}
// Retrieve from fixtures
if (TSID.startsWith("L") || TSID.startsWith("G"))
{
Expand Down

0 comments on commit 2baafa7

Please sign in to comment.