Refactor system operations config/state bucket (#545)

THIS VERSION REQUIRES UPDATING CLOUD FUNCTIONS
faucetsdn · Dec 24, 2022 · 381b90d · 381b90d
1 parent 8800e1a
commit 381b90d
Show file tree

Hide file tree

Showing 113 changed files with 4,372 additions and 743 deletions.
diff --git a/.gencode_hash.txt b/.gencode_hash.txt
@@ -1,27 +1,27 @@
 b35646a2f3d7c30fd34e3ea6ab6a1070b39c469fd1de49cab1093ff32faaf06d  gencode/docs/command_discovery.html
 3e292717e9b220403c250eebcfb4132becb6f59546f0bd3c3ec3042c31986d48  gencode/docs/command_mapping.html
-ca860612da2fddd2747d5a1f765efe9205be37b0ad4b327052320f915eb17948  gencode/docs/config.html
+64bafb080b454189b7b64a95e049c3427fd1cca70cf0809a31150610f40e9525  gencode/docs/config.html
 73b3beb740f5bc3ff26d173c356b2d3bf3b9892be83a704d3eeee7c9ff030230  gencode/docs/config_mapping.html
 b12d1fd583bc750b99aa343beea680d08fdc0110c6da8684cee24800c8cfaea0  gencode/docs/configuration_endpoint.html
 b7ca70e3176f997f254c32bd8d53dff1462ac8a71c0e166aaa85193d67bff60b  gencode/docs/configuration_execution.html
 4ac5b147d64b5a10408649dec791ac3b4d55a4c45f3f63489e1a980dbb1c4293  gencode/docs/configuration_pubber.html
 0f0355e343cb8197dfbae1481c0eca3a0566d73181bc968e33fd469a1c94d1e6  gencode/docs/envelope.html
-490e684c7710138b6c9fd41b131acf6638df64d715e35c799bd95a06756b7d87  gencode/docs/event.html
+92da453dec3e27493216f8c3672b6203be9eb9b3086741c22fda9432c22504b7  gencode/docs/event.html
 e81e06349dd9d88a57d7f6119da1fee2832b07fe8190b6df139e26777e7e76bc  gencode/docs/event_discovery.html
 9497b83603d537e11e264ee2d8056b298f5a5178e2c6aa435a6c9c0a051414b4  gencode/docs/event_mapping.html
 71fe25d3b17d1dc87b52c049a3235c20527f73e12ffa53fa4800a15e53f73732  gencode/docs/event_pointset.html
-3851c5c20c1938a14e21d7a4e329377b4582b7f88f1b18c0e9b34656e316e35e  gencode/docs/event_system.html
+c601733d04b0d389d7800a801b0862da41567d1f1fa63d9529a462b8269f0462  gencode/docs/event_system.html
 301d174a945f82816c9640450e072c55abdcb37383b76428e6d37dfbc6aa9dfe  gencode/docs/event_validation.html
 9ae793a641f2e053c5af734bf813cd2637a675330acb8d9e903ef03b4286e007  gencode/docs/metadata.html
 f01a413b528783682ecbc3b846d677490aa1e0a0cf15867e2305cee60a1f4a8e  gencode/docs/persistent_device.html
 5d039d607af9ec75ee552dfe36b16c702687ea16f5663f41fc49b4533b86e00d  gencode/docs/properties.html
 392f272c1c8cd047eb6906f2f11ffacc7083c32f639da2437d6a9ca861062ecc  gencode/docs/readme.md
-690d56e96c4e180e49e0abcba358ab1842ea06dd3f693918852a9de370c994d5  gencode/docs/reflect_config.html
+f46367beaab5886ef87ce70b542611134c3565daa53e367de4f49dab51214f12  gencode/docs/reflect_config.html
 9332e44c87dae9261b079424e748d5ee1df08a3c6b39987d254ebf78274e2f34  gencode/docs/reflect_state.html
 741b880216be3743f6747800a042f2dbd89f3b0344c6b0a965f4bc010f03a930  gencode/docs/schema_doc.css
 878ea88206c974f40643c3cc430875f9c4e8c5e3fd6bcd6358bd3eb6d48699a9  gencode/docs/schema_doc.min.js
 7ed934930aee763e0beebc349725ba3909115e8d346bb762f28bcbe745bb163a  gencode/docs/schema_extras.js
-a63c6d5ab8e3fd869d584578b5a850a86e43247f860d2830a4565da83f25b3e1  gencode/docs/state.html
+63cd94346f634ca2945982c147546cb1103fcfea4c42eac04b065ca838f5217f  gencode/docs/state.html
 f10172c9b4cd350cae1f673077b07b1362168599b966a78d4ec4ef19b7bbf441  gencode/docs/state_mapping.html
 552f85739f547960d2c1b96c99152ff831a036ce4d19908a5361b35a2d5f2132  gencode/docs/state_validation.html
 d39d7fe37a41c74a40080af7b0a429d201ab1fdff7444428c4b98eb7b38c332b  gencode/java/udmi/schema/Asset.java
@@ -71,7 +71,8 @@ b815ba1f198fd32b11fbedf71e5fa820bba08713dc5c95603c700dfa21ec5904  gencode/java/u
 052b6b7f9ca8173901d9d2fae314f5858e034f20efa5d184780b800bd870cb2b  gencode/java/udmi/schema/MappingEventEntity.java
 d2bf4eea0ca3df47b9ffe31481a52170e2d2bc3a0e7f2eab582e93cc20ccc886  gencode/java/udmi/schema/MappingState.java
 304164da05dc722b6e94cfa68659f0120e2425c94bfb5f5a4c6d796fe13da885  gencode/java/udmi/schema/Metadata.java
-bce55f08bba033c8dd05f63a91d99f86d2039f833130a1188eec8c774a115b50  gencode/java/udmi/schema/Metrics.java
+df64e4ddbf543ac70e7c2af9d3fbc20ffe3dff68c6718aa9ceadab7f64d3d171  gencode/java/udmi/schema/Metrics.java
+e49db649d7080e0b54cd7dc556bf58ff6afd1688c9394fdeeb5103b64ab91730  gencode/java/udmi/schema/Operation.java
 5e1c5411fae4d7c47391ceb5d19ae864fcd484df75ac6b6db39fd2d12647dec8  gencode/java/udmi/schema/Physical_tag.java
 d808259db6bbcd26ecf438844ce286d15d4750906be24588d97acfbe8a4ae315  gencode/java/udmi/schema/PointEnumerationEvent.java
 eb478a6ebd66b150da8d6dc8fdf05a10d09d6d0ea1df8a0578728703718c5551  gencode/java/udmi/schema/PointPointsetConfig.java
@@ -89,23 +90,25 @@ ca2e7566106818ca7e5190c8041eb86f0c9b3251b0bda8c3ea7ce11a0c891a0a  gencode/java/u
 997acde04cef70d3f6d009a21f3ce4f3d06e6c3093736c03fdc82d78ffea1aa8  gencode/java/udmi/schema/PubberOptions.java
 7bc53cb9a61df8f804a75aa4860403d19281f7c665cbddd5364a2aa746a1f651  gencode/java/udmi/schema/ReflectorConfig.java
 abe99dd74122c186403baa6982300a9d5968f8bbb7a67b1689104111b98f32fb  gencode/java/udmi/schema/ReflectorState.java
-5a44075bc03f2b9b2cc090f007fd1692832871f0981dcb02579d8dda96a96206  gencode/java/udmi/schema/SetupReflectorConfig.java
+362499688dd17f0596e5d6f064db0fe3bb3f3a25bdf01319624ba423de779220  gencode/java/udmi/schema/SetupReflectorConfig.java
 649c0291ad81421fc51da0c2f7da3286628127157a5a6eef77610e8c37c14941  gencode/java/udmi/schema/SetupReflectorState.java
 580df660dad1b97399002271716d597f72aa1a6110a49de9e162104c231752f4  gencode/java/udmi/schema/State.java
-b2274107ce0e4e663b2a35e2ca284e492ec683bd687f6b8d8b014d5f8be67233  gencode/java/udmi/schema/SystemConfig.java
-7de481ba531ea915a57cb6c5b23278db4fac6588354683cf63613b20c522af24  gencode/java/udmi/schema/SystemDiscoveryEvent.java
+47ad54328160a1aabce719a3263d4bfea903e8e8255e04a54ace86eb095b0f0c  gencode/java/udmi/schema/StateSystemHardware.java
+c9b920d355e5f6350fa6eb978afbfe55dd1438df171d2ea215845d5ac0b3e7e9  gencode/java/udmi/schema/StateSystemOperation.java
+d771dcb6bd06620e4e21419420d252b1f4aa0c55d0be315d6310299cebff56d6  gencode/java/udmi/schema/SystemConfig.java
+6f7b213970bfe2b1ac3056b83da8c8f5b4c50eac221dc1d6c6887e193e8bf40c  gencode/java/udmi/schema/SystemDiscoveryEvent.java
 21ca023d5a9006b1e041fcf644d572270b5743fcd2d2f221364a3046f0c5609c  gencode/java/udmi/schema/SystemEvent.java
 2cf23174ef4e2876511fb471d3f9fcb5cefe2fde324db844c2d0d505fd2c8844  gencode/java/udmi/schema/SystemHardware.java
 ceacfb4a1d0a4b4f4110b0e0717e57f2bbfd80ef18599d24fab19ddf497f399e  gencode/java/udmi/schema/SystemModel.java
-64e5b9a2c5699a3ca3b13dc6cb0243ff5b4c90908a14e3f11c5298ae89c8b261  gencode/java/udmi/schema/SystemState.java
+7d8d9f058878d3228a2042b8ffe6a5d897548c32b218307004101699aa17c2b3  gencode/java/udmi/schema/SystemState.java
 7d6dd13e368e7f073738fee69c15e18652a9b7d7ac63bde0a200f747e3aa1b1d  gencode/java/udmi/schema/TargetTestingModel.java
 d3968b92497e83a63f18cc0e74484a9807f1bb92db0c92d556ec2caaa143d645  gencode/java/udmi/schema/TestingModel.java
 7793d6d76a430dc7acf668d92d2df5f8e0625d6228207731f96e220f3b90e659  gencode/java/udmi/schema/TestingSystemConfig.java
 ac6f8fd87c8986cce01e872460c15ff6fe71e3816f9bde610acfe25f7d38c8d4  gencode/java/udmi/schema/ValidationEvent.java
 f7d117dc8b9764acf0c95a13a2bfdfbdf31d1a8ec83a707448aa4d7391ef07e2  gencode/java/udmi/schema/ValidationState.java
 e007ddd1ceeae3603c85110c33e1bb4a418ff9c7a791ca0df25b7ea3caeafd36  gencode/java/udmi/schema/ValidationSummary.java
 b77d953fd22e655c0f10ae32deeaa222769d971f8c38b3379eba45720fb910cc  gencode/java/udmi/schema/VirtualEquipmentLinks.java
-bef6dcfef0ed87cc87178b82c0ed702969c69d827aac6e69d45ff76518605e12  gencode/python/udmi/schema/__init__.py
+67553fdfe44896f487a7d4cf8161537e2595d48a96f8e44a1f9bac4a6d71591a  gencode/python/udmi/schema/__init__.py
 4b25dd95f863059b761269f93adcae7049507924a1c6e74d6856849203c179db  gencode/python/udmi/schema/ancillary_properties.py
 dab4f5fca272ec48c2881bca2b6bc43786ada47fa1f6dd935c35f7ce0eb6b0f6  gencode/python/udmi/schema/building_translation.py
 470b688984b89b25fcdfa8e08bd95b0c5d8c551d53a6ab5512503ee39419e6fa  gencode/python/udmi/schema/category.py
@@ -123,23 +126,23 @@ ac3facbd96f7cb2f7e387e7497d6a36af379a2687329571f250c5670f9933244  gencode/python
 1867ee8f1f4552ff121b7a2a62815817b2c9939a8817f1e093fbd688b8faa1a4  gencode/python/udmi/schema/config_mapping_device.py
 9eab64849e04b25203d5da47856c3f8dda2b96903e4dc43ab932ee35014700bd  gencode/python/udmi/schema/config_pointset.py
 33ce7ee99051d56dd3c4d69edf92583d0bb7fd751417948b71292ac2285d3c19  gencode/python/udmi/schema/config_pointset_point.py
-ba37d2d54df565aba42ef055a0ec961175d180c2e8092a914e62029bc5388857  gencode/python/udmi/schema/config_system.py
+7da3bdb37f338260d5f3829fa5fcbb9bbf9f146b514a68319c314a96c6b8ac12  gencode/python/udmi/schema/config_system.py
 cce623b34fd694880039a1c080214c33e00acaef5bc72276cf11a3bb2de40000  gencode/python/udmi/schema/config_system_testing.py
 cda883348ee365c981355c0bab1eea1745768a7dd03197ba1370813d95aca225  gencode/python/udmi/schema/configuration_endpoint.py
 e8a1574074554b6144b178d2adedc76a1c7be5ae911b253deff4460d8d82c6ee  gencode/python/udmi/schema/configuration_execution.py
 7f4f8c1a6ead34bf72b9134b97a39001752eb0f9647fafaac82ced4bb2f46b88  gencode/python/udmi/schema/configuration_pubber.py
 998ce105f88686f27b85f3630a396ed04b106f830c133a684ea5c505ca95b1c3  gencode/python/udmi/schema/envelope.py
 ed8ace0196c5e99d20778cb4210aa88cfe44a5e4539af98a1e194b9a71fa538e  gencode/python/udmi/schema/equipment_translation.py
 1eb9019b9d0b4ff7de2df8beb625a4f89292d636ece0c02f160495c537bd6c2c  gencode/python/udmi/schema/event.py
-aae3399ce27d070b553d6d1da8001ad65f61a8c1705bf2827e35b858ee637dae  gencode/python/udmi/schema/event_discovery.py
+bdc5ee3e4779f2dc28bafa57461aeea69bf02877d289848b669c55fa8aa570ec  gencode/python/udmi/schema/event_discovery.py
 ad33b91a7fabb4eed7e49c30a983a2106c96330facbe0f376f94d06e2263d6d0  gencode/python/udmi/schema/event_discovery_family.py
 42e6fb6edda2b6aeae5e5b48ae66f87a5daf336bca1e746b1157e8f1ce31d5d5  gencode/python/udmi/schema/event_discovery_point.py
 50bdecfa353d86d04a1ad0ebac5b3625f8fd4ee8a1e2f8e51b29055a37d9685c  gencode/python/udmi/schema/event_mapping.py
 6163270aa726ef29bc9285d224ec5c2611aab5d19f6ed09d711e6c816fc7e467  gencode/python/udmi/schema/event_mapping_entities.py
 3652a73a7df2d9df18a45b72d948c38a84a61e008d7543d006dad1832640c54f  gencode/python/udmi/schema/event_mapping_entity.py
 ddf849bfeb2b87d071cefd5e6feacabc57375a7fff6d72b6d42ffb89f33c859b  gencode/python/udmi/schema/event_pointset.py
 44aff1bc930dbdbadd51ac3fe0e7d9c83ad84a6a9f9d1c809b3fce66cbcd5e00  gencode/python/udmi/schema/event_pointset_point.py
-72b8c0c02951e5bbd37ad6fb8b8a035a45ed6ef1da642a664d517e86bb15d5a8  gencode/python/udmi/schema/event_system.py
+d89500d1e1c88ac754ddc5abd13135dd140d40c60568ca9294f6fe384b0b12ef  gencode/python/udmi/schema/event_system.py
 fb8a8a0e09eca4bb061c6cb2ff91b8eee77203e6ea353d3356e411692e1e4f2a  gencode/python/udmi/schema/event_validation.py
 2bb36986f805c2127f0c300de07bce07897c0c6d8f504bddec6e201d0f0ab1bf  gencode/python/udmi/schema/event_validation_device.py
 42f3429edf4a187be6ebfe9760f5e14821a25afa8f7c65b69e0e34fb29a7932e  gencode/python/udmi/schema/metadata.py
@@ -158,7 +161,7 @@ aafe6e70c281152db958adf77a024e3e9fab8293927106297c5ec48c11f54e27  gencode/python
 e8e694e260bb70998d228e14516b2fedb7e9d82e68f5586a7e78d0573bd2de8b  gencode/python/udmi/schema/options_pubber.py
 6c5f3dd1c5ca9d821e3c48298af118fc7eafd97af9265dfd34b2ed8642efca77  gencode/python/udmi/schema/persistent_device.py
 a58f8c98e837a5b56126ca0f410e02f1e9cfcd80a8cb429e0ef522defab1f690  gencode/python/udmi/schema/properties.py
-1f521678016ad267ad1c817896c7900ba30a85fc694669577b71148193db32c1  gencode/python/udmi/schema/reflect_config.py
+5dad0f2ee3e94ec32d11d1de3bfde35d087d3561ee18c0a9eeb0722d17d0f58a  gencode/python/udmi/schema/reflect_config.py
 1a4eef286957d77418777eb7f4cd1ee13aa24c2d916ec71a7ff4e56de76e303e  gencode/python/udmi/schema/reflect_state.py
 32bc70a30e37e89cfae14b44add18d546a6f9e00a3ec3519ede9c7486114d39c  gencode/python/udmi/schema/state.py
 4a908cee3fb8afb559bcbfa594e57dbc515a35e4468e02600751b2fcce05a238  gencode/python/udmi/schema/state_blobset.py
@@ -170,7 +173,8 @@ cb6a5558fe87dea5bce9e1cc6b22984a7d8c379babc4941c50df13edad36579a  gencode/python
 f2a76b32a5568d87a8c9ca8b32dc8a129bc7386f9522c5399d32db433be4cb70  gencode/python/udmi/schema/state_mapping_device.py
 3520ad936af70b414d9e7f90e606a011768bc4ee3bf1248714acc517ee9b393d  gencode/python/udmi/schema/state_pointset.py
 837ecc89c477abe3a1faf837733ca05475774891b55353d84ca231d90a1fbf31  gencode/python/udmi/schema/state_pointset_point.py
-8184e783f3e2da5a6d3fb7e7fac105aaeb28106a7c146e82b610049554abf184  gencode/python/udmi/schema/state_system.py
-791006619518fd7b38adb532879bcacd6f3f8795026cd75e10d3434a177757ac  gencode/python/udmi/schema/state_system_hardware.py
+8a601de0130081eec02724fd554749b7bd96d02874b0a3557d461cced41bd09b  gencode/python/udmi/schema/state_system.py
+8a0bf4294fb70533a056322a6bfc6f13a963381d2100d1468d8b6de757cd72e9  gencode/python/udmi/schema/state_system_hardware.py
+37c55dce0e38dddad3cebb3d40270128a4dbb7b586657d1746bb96dac90e2ac1  gencode/python/udmi/schema/state_system_operation.py
 231479e4dd7d961d59149580aba6a99cd466cea553400f338e9da2bf3ffbe78c  gencode/python/udmi/schema/state_validation.py
 83a57db63df10d43b1789c3f115a515c23bb7bc7a8cc8c2accdc17d0cbac89fe  gencode/python/udmi/schema/virtual_links.py
diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
@@ -24,17 +24,17 @@ jobs:
           python-version: '3.10'
       - name: Checkout source
         uses: actions/checkout@v2.3.4
-      - name: Install dependencies
+      - name: bin/run_tests install_dependencies
         run: bin/run_tests install_dependencies
-      - name: code checks
+      - name: bin/run_tests code_checks
         if: ${{ always() }}
         run: bin/run_tests code_checks
-      - name: Schema conformance tests
+      - name: bin/run_tests schema_tests
         if: ${{ always() }}
-        run: bin/run_tests schema_conformance_tests
-      - name: simple trace test
-        run: bin/run_tests simple_trace_test
-      - name: Registrar tests
+        run: bin/run_tests schema_tests
+      - name: bin/run_tests trace_tests
+        run: bin/run_tests trace_tests
+      - name: bin/run_tests registrar_tests
         if: ${{ always() }}
         run: bin/run_tests registrar_tests
 
@@ -92,7 +92,7 @@ jobs:
   udmi:
     name: Integration Tests
     runs-on: ubuntu-latest
-    timeout-minutes: 40
+    timeout-minutes: 30
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-java@v1
@@ -107,8 +107,6 @@ jobs:
         run: |
           bin/test_validator $GCP_TARGET_PROJECT
           more /tmp/validator.out
-          echo Comparing test run results with golden file:
-          diff -u /tmp/validator.out etc/validator.out
       - name: sequence tests
         env:
           GCP_TARGET_PROJECT: ${{ secrets.GCP_TARGET_PROJECT }}

diff --git a/bin/check_version b/bin/check_version
@@ -15,7 +15,7 @@ if [[ -z $origin ]]; then
     origin=faucetsdn
 fi
 
-git fetch $origin --tags 2>/dev/null
+git fetch $origin --tags 2>/dev/null || true
 
 upstream_ver=`git describe $origin/$branch --abbrev=0 || echo unknown`
 current_ver=`git describe HEAD --abbrev=0 || echo unknown`

diff --git a/bin/clone_model b/bin/clone_model
@@ -5,7 +5,7 @@ mkdir -p $ROOT_DIR/sites
 cd $ROOT_DIR/sites 
 
 MODEL_DIR=udmi_site_model
-MODEL_VER=1.8
+MODEL_VER=1.9
 TEST_SITE_GIT=https://github.com/faucetsdn/$MODEL_DIR.git
 MODEL_REPO=origin
 

diff --git a/bin/run_tests b/bin/run_tests
@@ -5,13 +5,13 @@ function usage() {
 
 $0 [step]
 
-step is one of:
-
+where [step] is one of:
   install_dependencies
   code_checks
-  schema_conformance_tests
-  simple_trace_test
+  schema_tests
+  trace_tests
   registrar_tests
+  all_tests
 EOF
   false
 }
@@ -27,16 +27,22 @@ case "$1" in
     validator/bin/build check
     bin/test_pylint
     ;;
-  schema_conformance_tests)
-    bin/test_schema -p
+  schema_tests)
+    bin/test_schema
     ;;
-  simple_trace_test)
+  trace_tests)
     bin/test_trace simple
     ;;
   registrar_tests)
     bin/test_registrar
     bin/test_sites
-    ;; 
+    ;;
+  all_tests)
+    bin/run_tests code_checks
+    bin/run_tests schema_tests
+    bin/run_tests trace_tests
+    bin/run_tests registrar_tests
+    ;;
   *)
     usage
     ;;

diff --git a/bin/test_redirect b/bin/test_redirect
@@ -70,7 +70,9 @@ sha256=$(openssl sha256 < out/endpoint.json | awk '{print $2}')
 cat <<EOF > out/blobs.json
 {
   "system": {
-    "mode": "active"
+    "operation": {
+      "mode": "active"
+    }
   },
   "blobset": {
     "blobs": {
@@ -86,7 +88,7 @@ EOF
 
 # Merge JSON files together into new redirect config
 jq -s '.[0] * .[1]' $out_base/generated_config.json out/blobs.json > $out_base/redirect_config.json
-jq '.system.mode = "restart"' $out_base/redirect_config.json > $out_base/restart_config.json
+jq '.system.operation.mode = "restart"' $out_base/redirect_config.json > $out_base/restart_config.json
 
 echo New redirection config:
 cat /tmp/${device_id}_config.json
@@ -107,7 +109,7 @@ sleep 20
 # Now test that pubber responds to an later last_start config (to trigger automatic shutdown)
 now_date=$(python3 -c 'import datetime; print(datetime.datetime.utcnow().isoformat() + "Z")')
 echo Setting last_start time to $now_date for pubber shutdown...
-jq ".system.last_start = \"$now_date\"" $out_base/redirect_config.json > $out_base/shutdown_config.json
+jq ".system.operation.last_start = \"$now_date\"" $out_base/redirect_config.json > $out_base/shutdown_config.json
 echo bin/reset_config $site_path $project_id $device_id shutdown_config.json
 bin/reset_config $site_path $project_id $device_id shutdown_config.json
 

diff --git a/bin/test_schema b/bin/test_schema
@@ -10,30 +10,30 @@ rm -f $errorfile
 
 build=y
 force=n
-parallel=
+upgrade=
 
 schemadir=schema
 testdir=tests
 
-while getopts "d:s:fnp" opt; do
+while getopts "d:s:fnu" opt; do
     case $opt in
         f)
             force=y
             ;;
         n)
             build=n
             ;;
+        u)
+            upgrade=-u
+            ;;
         s)
             subsets=${OPTARG}
             ;;
         d)
             testdir=${OPTARG}
             ;;
-        p)
-            parallel=y
-            ;;
         \?)
-            echo "Usage: $0 [-p] [-f] [-n] [-d TEST_DATA_DIR] [-s TEST_SUBSET]"
+            echo "Usage: $0 [-f] [-n] [-u] [-d TEST_DATA_DIR] [-s TEST_SUBSET]"
             exit -1
             ;;
     esac
@@ -48,10 +48,6 @@ fi
 
 jarfile=$(realpath validator/build/libs/validator-1.0-SNAPSHOT-all.jar)
 
-if [[ -n $parallel ]]; then
-    echo ========================= Running validation tests in parallel...
-fi
-
 if [[ -z $subsets ]]; then
     subsets=$(cd $testdir; ls -d *.tests)
 fi
@@ -63,6 +59,18 @@ echo Testing against $subsets
 for subset in $subsets; do
     schemaname=${subset%.tests}.json
     testfiles=$(cd $testdir/$subset; ls *.json || true)
+    testbase=$testdir/$subset
+
+    echo ================================== $subset
+
+    if [[ "$testbase" == "$schemadir" ]]; then
+        reltest=$subset
+    else
+        reldir=$(realpath --relative-to $schemadir $testdir)
+        reltest=$reldir:$subset
+    fi
+    (cd $schemadir; java -jar -Dnashorn.args=--no-deprecation-warning $jarfile -a $schemaname $upgrade -f $reltest) || true
+
     for testfile in $testfiles; do
         outfile=${testfile%.json}.out
         testbase=$testdir/$subset
@@ -88,27 +96,21 @@ for subset in $subsets; do
           reldir=$(realpath --relative-to $schemadir $testdir)
           reltest=$reldir:$(realpath --relative-to $testdir $testpath)
         fi
-        (
-            (cd $schemadir; java -jar -Dnashorn.args=--no-deprecation-warning $jarfile -a $schemaname -f $reltest) || true
-            if [ $force == y ]; then
-                diff $output $expected || echo Updating $expected && cp $output $expected
-            else
-                diff -b $output $expected || (echo '   ' diff $output $expected | tee -a $errorfile)
-            fi
-        ) &
 
-        if [[ -z $parallel ]]; then
-            wait
+        if [ $force == y ]; then
+            diff $output $expected || echo Updating $expected && cp $output $expected
+        else
+            diff -b $output $expected || (echo '   ' diff $output $expected | tee -a $errorfile)
+        fi
+
+        if [[ -n $upgrade ]]; then
+            upgrade_new=${output%.out}.json
+            diff $upgrade_new $testpath || (echo cp $upgrade_new $testpath | tee -a $errorfile)
         fi
     done
 done
 
-if [[ -n $parallel ]]; then
-    sleep 5
-    echo ========================= Waiting for validation results...
-    wait
-fi
-
+echo ==================================
 echo
 
 if [ -f $errorfile ]; then