Reference data set and generator

README.md

@@ -43,6 +43,8 @@ __IMPORTANT NOTICE:__ The contents of this repository currectly reflect a __DRAF
    1. [Confidence Level Joining](./usage-examples/confidence-level-joining.md)
    1. [Delivery Interface](./usage-examples/delivery-interface.md)
    1. [Build Avoidance](./usage-examples/build-avoidance.md)
+   1. Reference Data Sets
+      1. [Default](./usage-examples/reference-data-sets/default.md)
 1. Customization
    1. [Custom Events](./customization/custom-events.md)
    1. [Custom Data](./customization/custom-data.md)

examples/flows/confidence-level-joining/events.json

@@ -111,7 +111,7 @@
     },
     "data": {
       "name": "Act1",
-      "category": "Test Activity",
+      "categories": ["Test Activity"],
       "triggers": [
         {
           "type": "EIFFEL_EVENT"
@@ -138,7 +138,7 @@
     },
     "data": {
       "name": "Act2",
-      "category": "Test Activity",
+      "categories": ["Test Activity"],
       "triggers": [
         {
           "type": "EIFFEL_EVENT"
@@ -401,7 +401,7 @@
     },
     "data": {
       "outcome": {
-        "conclusion": "SUCCESS"
+        "conclusion": "SUCCESSFUL"
       }
     },
     "links": [

examples/validate.py

@@ -5,51 +5,89 @@
 import fnmatch
 from jsonschema import validate
 
-def applySchema(schemaFileName):
-  print("  - Applying", schemaFileName, "to ...")
-  global schemas
-  schemas +=1
-  eventTypeName = schemaFileName[:-5]
-  eventTypeDirName = "examples/events/" + eventTypeName
-  try:
-    with open("schemas/" + schemaFileName, "r") as f:
-      schema = json.load(f)
+def loadAllJsonObjects(dir):
+  objects = []
+  badFiles = []
+
+  for root, dirNames, fileNames in os.walk(dir):
+    for fileName in fnmatch.filter(fileNames, "*.json"):
+      try:
+        path = os.path.join(root, fileName)
+        with open(path, "r") as f:
+          loadedObject = json.load(f)
+          if(isinstance(loadedObject, list)):
+            for o in loadedObject:
+              objects.append((path, fileName, o))
+          else:
+            objects.append((path, fileName, loadedObject))
+      except Exception as e:
+        print(e)
+        badFiles.append(path)
+
+  return objects, badFiles
+
+def loadSchemas():
+  schemaTuples, badSchemaFiles = loadAllJsonObjects("schemas")
+  schemas = {}
+  for path, fileName, o in schemaTuples:
+    schemas[fileName[:-5]] = o
+  return schemas, badSchemaFiles
 
-    for root, dirnames, filenames in os.walk(eventTypeDirName):
-      for exampleFileName in fnmatch.filter(filenames, "*.json"):
-        validateExample(schema, os.path.join(root, exampleFileName))
-  except Exception as e:
-    reportFailure(e)
-
-def validateExample(schema, exampleFilePath):
-  print("    ... ", exampleFilePath)
-  global examples
-  examples +=1
-  exception = False
-  try:
-    with open(exampleFilePath, "r") as f:
-      example = json.load(f)
+def loadExamples():
+  exampleTuples, badExampleFiles = loadAllJsonObjects("examples")
+  examples = []
+  for path, fileName, o in exampleTuples:
+    examples.append((path, o["meta"]["type"], o["meta"]["id"], o))
+  return examples, badExampleFiles
+
+def validateExamples(examples, schemas):
+  failures = []
+  numberOfSuccessfulValidations = 0
+  unchecked = []
+
+  for path, type, id, json in examples:
+    if type in schemas:
+      try:
+        validate(json, schemas[type])
+        numberOfSuccessfulValidations += 1
+      except Exception as e:
+        failures.append((path, type, id, e))
+    else:
+      unchecked.append((path, type, id, json))
 
-    validate(example, schema)
-    print("         PASS")
-  except Exception as e:
-    reportFailure(e)
+  return failures, unchecked, numberOfSuccessfulValidations
+
+def report(unchecked,failures,badSchemaFiles,badExampleFiles,numberOfSuccessfulValidations):
+  for path, type, id, o in unchecked:
+    print("WARNING: Missing schema for " + id + "(" + type + ") in " + path + ".")
+
+  for badSchemaFile in badSchemaFiles:
+    print("ERROR: Failed to load schema from file", badSchemaFile)
+
+  for badExampleFile in badExampleFiles:
+    print("ERROR: Failed to load example from file", badExampleFile)
+
+  for path, type, id, e in failures:
+    print("ERROR: Validation failed for " + id + "(" + type + ") in " + path + ": " + str(e))
 
-def reportFailure(exception):
-    global failures
-    failures += 1
-    print("         FAIL:", type(exception).__name__)
-    print("        ", exception)
+  print("")
+  print("===SUMMARY===")
+  print("Bad schema files:        ", len(badSchemaFiles))
+  print("Bad example files:       ", len(badExampleFiles))
+  print("Successful validations:  ", numberOfSuccessfulValidations)
+  print("Failed validations:      ", len(failures))
+  print("Unchecked examples:      ", len(unchecked))
+  print("=============")
+
+schemas, badSchemaFiles = loadSchemas()
+print("Loaded", len(schemas), "schemas.")
 
-failures = 0
-schemas = 0
-examples = 0
+examples, badExampleFiles = loadExamples()
+print("Loaded", len(examples), "examples.")
 
-for root, dirNames, fileNames in os.walk("schemas"):
-    for schemaFileName in fnmatch.filter(fileNames, "*.json"):
-      applySchema(schemaFileName)
+failures, unchecked, numberOfSuccessfulValidations = validateExamples(examples, schemas)
 
-print("Encountered", failures, "validation failures through application of", schemas, "schemas to", examples, "examples.")
+report(unchecked, failures, badSchemaFiles, badExampleFiles, numberOfSuccessfulValidations)
 
-if failures > 0:
+if len(badSchemaFiles) > 0 or len(badExampleFiles) > 0 or len(failures) > 0:
   sys.exit("Validation failed.")

usage-examples/reference-data-sets/default.md

@@ -0,0 +1,8 @@
+# Default Reference Data Set
+This is a description of the "Default" reference data set found [here](../../examples/reference-data-sets/default/events.zip).
+
+The purpose of a reference data set is to serve as supporting documentation, but also to allow for functional and non-functional testing as well as benchmarking of implementations.
+
+This particular data set is designed as a combination of three usage examples: [Confidence Level Joining](../confidence-level-joining.md), [Delivery Interface](../delivery-interface.md) and [Build Avoidance](../build-avoidance.md) and consists of multiple iterations through the represented continuous integration and delivery system, with certain random elements (e.g. sometimes tests fail, sometimes tests succeed). A single iteration is represented graphically below.
+![alt text](./default.png "Single iteration of the Default Reference Data Set")
+