Automatically generate category constants for Java and schema validation

.gencode_hash.txt

@@ -1,33 +1,34 @@
-f5cf191b0151d2142fa634d1deafb93e35041d5b9ae89b26beab32fa25fb9521  gencode/docs/config.html
-90679d3d866579501e7aa00b515af05d42fc9fe399eafacaacf297d1e4a22884  gencode/docs/envelope.html
-368f12b9b695b31d97e65509c461f56c12762ec50b9baff89fd7fd85713e8fce  gencode/docs/event_discovery.html
+7ce008da1ec22a6dc9d4ca3ddc0f4a5b0c8ff6efca8e781edf0720676c9b8115  gencode/docs/config.html
+e5f11a58d744c5f82c4156e2c82b67a3424f0bf5157a6c666f80a3c6d46d2aa7  gencode/docs/envelope.html
+9362682fa830ce9d98b3c074cab05ba44fae96a82de316a3520df3956e9325dc  gencode/docs/event_discovery.html
 8133e380e40f27c56accbffc665b2eeb56ec84a4da3b52ba7aa5e439c9c40572  gencode/docs/event_pointset.html
-076eb51e75281ed065a9f0236a1e19b6bf3c277b51ceca84f6aa3f76d5bb6022  gencode/docs/event_system.html
-465a940ab96c32e040ae14b62f384ae9c472f646f3de1ea96288fd4c11de1409  gencode/docs/event_validation.html
+06a8831025ad3cba59ed1831842db537525c26a713de92f6dd6c48656e3d772d  gencode/docs/event_system.html
+a471b2ded89ceb93dd4c70e9702db179fdd517fd45bfd2b43a95f8b9f287b50b  gencode/docs/event_validation.html
 f48026471ae3cd7867bce416dc21c2fb728f48d8476a8d6e95f6acaf1d8b6cf3  gencode/docs/index.html
 6d5081d04c86f8bd444efb41cd788826aa66802d37e8f98e93b6b75425542d5e  gencode/docs/metadata.html
 741b880216be3743f6747800a042f2dbd89f3b0344c6b0a965f4bc010f03a930  gencode/docs/schema_doc.css
 878ea88206c974f40643c3cc430875f9c4e8c5e3fd6bcd6358bd3eb6d48699a9  gencode/docs/schema_doc.min.js
 7ed934930aee763e0beebc349725ba3909115e8d346bb762f28bcbe745bb163a  gencode/docs/schema_extras.js
-16d46783ef2271abef54f071bb3723a4d124a95ec4fb5a536a7dee5c266d8899  gencode/docs/state.html
+ffe0596792d14a35c395f84d5ee1ec76032fd29e0dd9c440984db8f8a5f1be1d  gencode/docs/state.html
 d39d7fe37a41c74a40080af7b0a429d201ab1fdff7444428c4b98eb7b38c332b  gencode/java/udmi/schema/Asset.java
 0825a5cec83003bb0a6488c4ed7010a04ae0d3848ef36fe01bb4e6718ba7b96d  gencode/java/udmi/schema/Aux.java
 d326a550d75ab7f558f165f1a5e8c24eee2b0b52290a7472f529bb1e15d79fb2  gencode/java/udmi/schema/BlobBlobsetConfig.java
 2c03651cb2ecda072b1418222eebb5560185669f8ffdd03021ad5ad8ff7ba3b0  gencode/java/udmi/schema/BlobBlobsetState.java
 3fccb6bc6714fdc3dd912c8b1155e7ce19b0ac8a7ec48a01be9eb27bdd034a75  gencode/java/udmi/schema/BlobsetConfig.java
 fcbed49f1af8b791d8c52bcbe18f65521a79d9ac3eb33ec3afd9b342ab2bfc56  gencode/java/udmi/schema/BlobsetState.java
+0c908ea34ee54aa66f9ce472ad23ecaec3c091f2afbba7b509e1515b5557f82c  gencode/java/udmi/schema/Category.java
 d6875f63ce67d1b945a0b75a4a660bd083cc52492371a7350c4109f6bf54968b  gencode/java/udmi/schema/CloudIotConfig.java
 a2eeff86f4302272736d84602e2ca36a64d27c8ef6761cc05ffb8ad17b030d4d  gencode/java/udmi/schema/CloudModel.java
-ff79de9390aa25bb45fb3e2ebb682c865ccab764f56d9644377d9d28c0ab10e4  gencode/java/udmi/schema/Config.java
+285d6d221baf084d35adf16f3e4bddbed2fb60b9c7b9bfca8adcdcb035fb929c  gencode/java/udmi/schema/Config.java
 d4a82e132b2d223c4a82384d22455283c0b64aba3100fc210e72f9700b9df2d0  gencode/java/udmi/schema/DeviceValidationEvent.java
 587d67a67431349939dffd37b880c44e798a1eb607d54bd6d8a077bbac668067  gencode/java/udmi/schema/DiscoveryCommand.java
 d8a80ab3180d33bfa17564c969018e1d4350a47dbc70c4ae8a5abbfb25cfedc9  gencode/java/udmi/schema/DiscoveryConfig.java
 4793b2370b1f3b8fd89941586b779dffdde83d321a2621907f67d4d297d19b17  gencode/java/udmi/schema/DiscoveryEvent.java
 04112dd47b0f761131c276c67d3cd8b789d25e6716b5732be9fef14fc6831f1d  gencode/java/udmi/schema/DiscoveryModel.java
 9962b0eb7d5adf52af6160e9f3131f8eeb52ae9e518954dbb6aead1bcad0245e  gencode/java/udmi/schema/DiscoveryState.java
 4e64cf10484266a297fb418b165dd84a9f4b8c2e53520b33915cc96365151fc2  gencode/java/udmi/schema/EndpointConfiguration.java
-090bbaf1508aa6ca8380af936af990673f300eb5a940c9e8ab94deb64efa2b7b  gencode/java/udmi/schema/Entry.java
-cd362f94454eba8fd5ce3fce5d5e2b5f046d0dd9c35b01de69ef4d2e38413cc5  gencode/java/udmi/schema/Envelope.java
+885859ea9de29217186c34f3a90b112411d1beddaf58a88b85e7934cd3c92b6f  gencode/java/udmi/schema/Entry.java
+db72638046c1e370094ae8128d5b9b7cab5a2ddedfc2b9b14203d74eb03ef177  gencode/java/udmi/schema/Envelope.java
 e9f5c77be81486b6b8c6d88f70f2d50583d8c3fafa2ac09ead80f44b8d5e751e  gencode/java/udmi/schema/Event.java
 aa0885ca43ab38c7597eacc38b7c512940a1a9fa061abd47d02c28e66b6fd93e  gencode/java/udmi/schema/FamilyDiscoveryConfig.java
 ae4a645f199c8e24b3303463d428ca17af7603ae9ae9238397a6a82e752ab454  gencode/java/udmi/schema/FamilyDiscoveryEvent.java
@@ -57,7 +58,7 @@ fc3a9415c04d8a06954dbdbfdff5d68ab113cce3948532c19df555778ffb04fa  gencode/java/u
 ca2e7566106818ca7e5190c8041eb86f0c9b3251b0bda8c3ea7ce11a0c891a0a  gencode/java/udmi/schema/Position.java
 3df66bb1a37a9e0b2b6cf392f8c64d404a73c83e5e13c02bb4844f09b9a04b70  gencode/java/udmi/schema/Properties.java
 faa1cd11467f9e58742cfee7124d4af04dbf481eb5a80549bbd2eb77bcc8059d  gencode/java/udmi/schema/PubberConfiguration.java
-e767077a73430254f7365dbb3f242a1bb5a953a4016c116663d31ae3fbf00509  gencode/java/udmi/schema/PubberOptions.java
+a5737c70324cf13322c019049f80e334bf699dea2f434859762fe3553fee8308  gencode/java/udmi/schema/PubberOptions.java
 7bc53cb9a61df8f804a75aa4860403d19281f7c665cbddd5364a2aa746a1f651  gencode/java/udmi/schema/ReflectorConfig.java
 abe99dd74122c186403baa6982300a9d5968f8bbb7a67b1689104111b98f32fb  gencode/java/udmi/schema/ReflectorState.java
 5a44075bc03f2b9b2cc090f007fd1692832871f0981dcb02579d8dda96a96206  gencode/java/udmi/schema/SetupReflectorConfig.java
@@ -70,15 +71,17 @@ e0f70f8e2c4793ebd05a38dc249d329eaa5b0171097ecc5fad4630fa0df3536d  gencode/java/u
 1c79bdb3da4c9127c75aa6660be0e62bfab1d1ee47c289637662fee2e37f1491  gencode/java/udmi/schema/SystemModel.java
 127cd1f0f137cd79c8f42e289bd80e8509b7cc269f69d9ac10874706a7a1a0b6  gencode/java/udmi/schema/SystemState.java
 7d6dd13e368e7f073738fee69c15e18652a9b7d7ac63bde0a200f747e3aa1b1d  gencode/java/udmi/schema/TargetTestingModel.java
+3360be8c937bd8cc592f8c67abdf1b240175a1748aa8e321b3360896961967e6  gencode/java/udmi/schema/TestingConfig.java
 d3968b92497e83a63f18cc0e74484a9807f1bb92db0c92d556ec2caaa143d645  gencode/java/udmi/schema/TestingModel.java
 347da3d586ff6e5dcf3ce8ec0c800ec0efd5fed4e0c80eef35de65ad92ae48bf  gencode/java/udmi/schema/ValidationEvent.java
 e007ddd1ceeae3603c85110c33e1bb4a418ff9c7a791ca0df25b7ea3caeafd36  gencode/java/udmi/schema/ValidationSummary.java
-32e82ac7f7e20c77ab5d40d3de0a06c3934267d449b243bb9cad48493974c306  gencode/python/udmi/schema/__init__.py
+8696129c6e2b0b06779986e76657af367c9d1b289659e326edca7d47f35eface  gencode/python/udmi/schema/__init__.py
 4b25dd95f863059b761269f93adcae7049507924a1c6e74d6856849203c179db  gencode/python/udmi/schema/ancillary_properties.py
+70f9b662321b8acded5943749539fb9376f1247e986918ea2cfac65e3d307953  gencode/python/udmi/schema/category.py
 a61b1c5732f01b7efda41a773d5786fad755f371193ce4478b458387ca2a8fe8  gencode/python/udmi/schema/cloud_iot_config.py
 6578d68f65b87b781086e72566de910db4bef365599fe3188862d4d8a81e84fb  gencode/python/udmi/schema/command_discovery.py
 2082a9f3ff4d37bb13a647932fcacb99499f71569cd9dc12d5e63dead34fee63  gencode/python/udmi/schema/common.py
-b975892df78076dabc797b4c0be87f20b33eacda11f9d1ac1c09be33d4937a87  gencode/python/udmi/schema/config.py
+03ea5f3a917a2e809d03a6435eeb9a8369a8b489605ac50d9324bb8256896c38  gencode/python/udmi/schema/config.py
 79eb0299b3751d93c01a5de65eacc717283e99caf0996f3db48a15247f69c8eb  gencode/python/udmi/schema/config_blobset.py
 95125e3459cec6e12f4e0d3912f9c519b1aec41cf1df9720b54481e69632d7f4  gencode/python/udmi/schema/config_blobset_blob.py
 ec6c6ab1fb0f37a29b7ebd162aa77da7f1e261e80da376942a3b39d17ccf1be4  gencode/python/udmi/schema/config_discovery.py
@@ -88,6 +91,7 @@ b461bdc24310ef972faf579b5be577b5af67fb0977d6afb4c42955211b26e3d5  gencode/python
 9eab64849e04b25203d5da47856c3f8dda2b96903e4dc43ab932ee35014700bd  gencode/python/udmi/schema/config_pointset.py
 607c5047df878a1333df3ce88dcce34668959b0b315f6954bf1a4963dcf7839e  gencode/python/udmi/schema/config_pointset_point.py
 68685118163ea55f23938aafd10308faad1e2bf86c4f2fa9341cb53e19e2a260  gencode/python/udmi/schema/config_system.py
+b2f0867df30eeb3232674c34e2794297734766383736b81ac77bf60740b58c91  gencode/python/udmi/schema/config_testing.py
 998ce105f88686f27b85f3630a396ed04b106f830c133a684ea5c505ca95b1c3  gencode/python/udmi/schema/envelope.py
 1eb9019b9d0b4ff7de2df8beb625a4f89292d636ece0c02f160495c537bd6c2c  gencode/python/udmi/schema/event.py
 82182e3f569ad80dc0751027959c7db9135d10072fbe79f896d63a4cd2f4771f  gencode/python/udmi/schema/event_discovery.py
@@ -113,7 +117,7 @@ aafe6e70c281152db958adf77a024e3e9fab8293927106297c5ec48c11f54e27  gencode/python
 5c50847e136a033ea511209238bb570499b43fbee6189dae06603132dcb9f01f  gencode/python/udmi/schema/model_testing_target.py
 a58f8c98e837a5b56126ca0f410e02f1e9cfcd80a8cb429e0ef522defab1f690  gencode/python/udmi/schema/properties.py
 1b2dba2ff079366560c7106aecc5289db19312816fa34a86459e429c94972761  gencode/python/udmi/schema/pubber_configuration.py
-68b34e6aed281e62e2b62d0aaea57a57f1d3a657ed39cbd849ddd69659d9d454  gencode/python/udmi/schema/pubber_options.py
+0c895896f4bad5b50a9332706adb0f751618c37a125dbe6780ff92bd729923d7  gencode/python/udmi/schema/pubber_options.py
 1f521678016ad267ad1c817896c7900ba30a85fc694669577b71148193db32c1  gencode/python/udmi/schema/reflect_config.py
 1a4eef286957d77418777eb7f4cd1ee13aa24c2d916ec71a7ff4e56de76e303e  gencode/python/udmi/schema/reflect_state.py
 32bc70a30e37e89cfae14b44add18d546a6f9e00a3ec3519ede9c7486114d39c  gencode/python/udmi/schema/state.py

.github/workflows/testing.yml

@@ -128,4 +128,4 @@ jobs:
         if: ${{ always() }}
         run: |
           cat pubber.out || true
-          more out/*.json
+          more pubber/out/*.json

bin/gencode_categories

@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+
+import hashlib
+import json
+import re
+import os
+import shutil
+import sys
+
+GENCODE_MARKER = '@@ '
+CATEGORY_MARKER = '* '
+CATEGORY_REGEX = ' *\\* _([a-z]+)_: (\(\*\*([A-Z]+)\*\*\) )?(.*)'
+JSON_FORMAT = '%s%s{ "pattern": "^%s$" }'
+
+JAVA_DESCRIPTION = "\n%s// %s\n"
+JAVA_TARGET = '%spublic static final String %s = "%s";\n'
+JAVA_LEVEL = '%spublic static final Level %s_LEVEL = %s;\n'
+JAVA_MAP_ADD = '%sstatic { LEVEL.put(%s, %s); }\n'
+
+doc_in = os.path.join('docs/specs/categories.md')
+schema_in = os.path.join('etc/category.json')
+schema_out = os.path.join('schema/category.json')
+java_in = os.path.join('etc/Category.java')
+java_out = os.path.join('gencode/java/udmi/schema/Category.java')
+
+def read_categories():
+    categories = []
+    prefix = []
+    previous = -1
+    group = None
+    with open(doc_in) as doc:
+        while line := doc.readline():
+            indent = line.find(CATEGORY_MARKER)//2
+            match = re.match(CATEGORY_REGEX, line)
+            if indent < 0 or not match:
+                continue
+            if indent < previous:
+                for _ in range(indent, previous):
+                    rem = prefix.pop(len(prefix) - 1)
+            elif indent > previous:
+                if group:
+                    prefix.append(group)
+            previous = indent
+            group = match.group(1)
+            category = '.'.join(prefix + [group])
+            level = match.group(3)
+            description = match.group(4)
+            if level:
+                entry = (category, level, description)
+                categories.append(entry)
+    return categories
+
+def write_schema_out(categories):
+    with open(schema_in) as inp:
+        with open(schema_out, 'w') as out:
+            while line := inp.readline():
+                index = line.find(GENCODE_MARKER)
+                if index > 0:
+                    write_schema_categories(out, line[0:index], categories)
+                else:
+                    out.write(line)
+
+def write_schema_categories(out, indent, categories):
+    prefix = ''
+    for category in categories:
+        target = category[0].replace('.', '\\\\.')
+        out.write(JSON_FORMAT % (prefix, indent, target))
+        prefix = ',\n'
+    out.write('\n')
+
+def write_java_out(categories):
+    os.makedirs(os.path.dirname(java_out), exist_ok=True)
+    with open(java_in) as inp:
+        with open(java_out, 'w') as out:
+            while line := inp.readline():
+                if line.strip().startswith(GENCODE_MARKER):
+                    indent = line[0:line.find(GENCODE_MARKER)]
+                    write_java_categories(out, indent, categories)
+                else:
+                    out.write(line)
+
+def write_java_categories(out, indent, categories):
+    for category in categories:
+        target = category[0]
+        level = category[1]
+        desc = category[2]
+        const = target.replace('.', '_').upper()
+        out.write(JAVA_DESCRIPTION % (indent, desc))
+        out.write(JAVA_TARGET % (indent, const, target))
+        out.write(JAVA_LEVEL % (indent, const, level))
+        out.write(JAVA_MAP_ADD % (indent, const, level))
+
+
+categories = read_categories()
+write_schema_out(categories)
+write_java_out(categories)

bin/gencode_docs

@@ -19,15 +19,21 @@
 ROOT_DIR=$(realpath $(dirname $0)/..)
 cd $ROOT_DIR
 
+OP=
+if [[ -n $1 ]]; then
+    OP=$1
+    shift
+fi
+
 OUTPUT_DIR=$ROOT_DIR/gencode/docs
 TMP_DIR=$ROOT_DIR/tmp/schema
 
 schema_files=`ls schema/*.json`
 
-if [ "$1" == gen_bucket ]; then
-    shift 1
+if [ "$OP" == gen_bucket ]; then
+    rm -rf $OUTPUT_DIR $TMP_DIR
 
-    input_file=$1
+    input_file=$OP
     output_file=$2
     tmp_file=$output_file.tmp
 
@@ -50,7 +56,7 @@ if [ "$1" == gen_bucket ]; then
 fi
 
 
-if [ "$1" == check_links ]; then
+if [ "$OP" == check_links ]; then
     if [ -d venv ]; then
         source venv/bin/activate
     fi
@@ -71,7 +77,6 @@ if [ -d venv ]; then
     source venv/bin/activate
 fi
 
-rm -rf $OUTPUT_DIR
 mkdir -p $OUTPUT_DIR
 
 # Make a copy of the schema files into a tmp folder for any pre generator fixes

bin/gencode_java

@@ -6,9 +6,11 @@ JBASENAME=jsonschema2pojo
 ROOT_DIR=$(realpath $(dirname $0)/..)
 cd $ROOT_DIR
 
-rm -rf gencode
+rm -rf gencode/java codegen/java
 mkdir -p codegen gencode
 
+bin/gencode_categories
+
 OUTDIR=$ROOT_DIR/gencode/java
 cd codegen
 GENDIR=$PWD

bin/gencode_python

@@ -3,6 +3,7 @@
 import hashlib
 import json
 import os
+import shutil
 import sys
 
 FILE_PREFIX="file:"
@@ -261,6 +262,7 @@ def generate_class(contents):
         write_class(writer, contents)
 
 def generate_module(in_files):
+    shutil.rmtree(out_dir)
     os.makedirs(out_dir, exist_ok=True)
     full_path = os.path.join(out_dir, '__init__.py')
     with open(full_path, 'w') as fd:

bin/pubber

@@ -18,6 +18,9 @@ if [ ! -f $site_path/cloud_iot_config.json ]; then
     false
 fi
 
+echo Cleaning output direction $ROOT_DIR/pubber/out
+rm -rf $ROOT_DIR/pubber/out
+
 echo Building pubber...
 $ROOT_DIR/pubber/bin/build
 

bin/test_sequencer

@@ -55,7 +55,7 @@ if [[ $i -eq $WAITING ]]; then
     false
 fi
 
-bin/sequencer $site_path $project_id $device_id $serial_no
+bin/sequencer -vv $site_path $project_id $device_id $serial_no
 
 sed -i -e 's/.*sequencer RESULT/RESULT/' /tmp/sequencer.out
 

docs/specs/categories.md

@@ -10,9 +10,9 @@ implicit expected `level` values, indicated by '(**LEVEL**)' in the hierarchy be
 
 * _system_: Basic system operation
   * _base_: Baseline system operational messages
-    * _start_ (**NOTICE**): System is in the process of (re)starting and essentially offline
-    * _shutdown_ (**NOTICE**): System is shutting down
-    * _ready_: (**NOTICE**): System is fully ready for operation
+    * _start_: (**NOTICE**) System is in the process of (re)starting and essentially offline
+    * _shutdown_: (**NOTICE**) System is shutting down
+    * _ready_: (**NOTICE**) System is fully ready for operation
     * _comms_: Baseline message handling
   * _config_: Configuration message handling
     * _receive_: (**DEBUG**) Receiving a config message
@@ -26,9 +26,21 @@ implicit expected `level` values, indicated by '(**LEVEL**)' in the hierarchy be
     * _logout_: (**NOTICE**) Successful logout 
     * _fail_: (**WARNING**) Failed authentication attempt. The entry message should include the application
 * _pointset_: Handling managing data point conditions
-  * _point_: Conditions relating to a specific point, the entry `message` field should
-  start with "Point _pointname_" followed by descriptive information.
-    * _applied_ (**INFO**): The `set_value` for a point has been implied
-    * _updating_ (**NOTICE**): The point is in the process of updating
-    * _overridden_ (**WARNING**): The reported value has been overridden locally
-    * _invalid_ (**ERROR**): A `config` parameter for the point is invalid in some way
+  * _point_: Conditions relating to a specific point, the entry `message` should start with "Point _pointname_"
+    * _applied_: (**INFO**) The `set_value` for a point has been applied
+    * _updating_: (**NOTICE**) The point is in the process of updating
+    * _overridden_: (**WARNING**) The reported value has been overridden locally
+    * _failure_: (**ERROR**) The system failed to read/write the point
+    * _invalid_: (**ERROR**) A `config` parameter for the point is invalid in some way
+* _discovery_: Handling on-prem discovery flow
+  * _family_: Conditions specific to an entire address family (e.g. bacnet)
+    * _scan_: (**INFO**) Relating to scanning a particular address family
+  * _device_: Conditions specific to device scanning
+    * _enumerate_: (**INFO**) Handling point enumeration for a given device
+  * _point_: Conditions specific to point enumeration
+    * _describe_: (**INFO**) Relating to describing a particular point
+* _blobset_: Handling update of device data blobs
+  * _blob_: Conditions specific to an individual blob
+    * _received_: (**DEBUG**) Request for an update has been received
+    * _fetched_: (**DEBUG**) Update blob has been successfully fetched
+    * _applied_: (**NOTICE**) Update has been successfully applied

etc/Category.java

@@ -0,0 +1,18 @@
+package udmi.schema;
+
+import static udmi.schema.Level.DEBUG;
+import static udmi.schema.Level.INFO;
+import static udmi.schema.Level.NOTICE;
+import static udmi.schema.Level.WARNING;
+import static udmi.schema.Level.ERROR;
+
+import java.util.HashMap;
+import java.util.Map;
+
+// This class is manually curated, auto-generated, and then copied into the gencode directory.
+// Look for the proper source and don't be fooled! Ultimately sourced from docs/specs/categories.md
+public class Category {
+    public static final Map<String, Level> LEVEL = new HashMap();
+
+    @@ gencode stuff goes here
+}

etc/category.json

@@ -0,0 +1,6 @@
+{
+  "$comment": "Auto-generated category mappings from bin/gencode_categories.",
+  "oneOf": [
+    @@ gencode stuff goes here
+    ]
+}