Gencode to include inline examples from test directory

.gencode_hash.txt

@@ -1,25 +1,16 @@
-a9cf25fa459ab0122be8df276ab4d95c6b1f2ed2256e1cf202f18c178e6906c0  gencode/docs/config.html
-90679d3d866579501e7aa00b515af05d42fc9fe399eafacaacf297d1e4a22884  gencode/docs/envelope.html
-80bbcb648a4ebe8f1b5dbb363e1654dd6db0cec2d67ac2c642f2bddc69d5a6ff  gencode/docs/event_discovery.html
-8133e380e40f27c56accbffc665b2eeb56ec84a4da3b52ba7aa5e439c9c40572  gencode/docs/event_pointset.html
-cd7622cc8f952aa5aa8b243ef266f1a7d6a311886913f884ffd80cfbcd97daa7  gencode/docs/event_system.html
-77f401d6ad8958cd299072b42d3714f3b71512680e4f948d197795da9c2baa44  gencode/docs/event_validation.html
-f48026471ae3cd7867bce416dc21c2fb728f48d8476a8d6e95f6acaf1d8b6cf3  gencode/docs/index.html
-6d5081d04c86f8bd444efb41cd788826aa66802d37e8f98e93b6b75425542d5e  gencode/docs/metadata.html
-741b880216be3743f6747800a042f2dbd89f3b0344c6b0a965f4bc010f03a930  gencode/docs/schema_doc.css
-878ea88206c974f40643c3cc430875f9c4e8c5e3fd6bcd6358bd3eb6d48699a9  gencode/docs/schema_doc.min.js
-7ed934930aee763e0beebc349725ba3909115e8d346bb762f28bcbe745bb163a  gencode/docs/schema_extras.js
-9b1f7c10c78d58acf650d8f500e8d972af17fce409189652cb62a1774760cd27  gencode/docs/state.html
 d39d7fe37a41c74a40080af7b0a429d201ab1fdff7444428c4b98eb7b38c332b  gencode/java/udmi/schema/Asset.java
 0825a5cec83003bb0a6488c4ed7010a04ae0d3848ef36fe01bb4e6718ba7b96d  gencode/java/udmi/schema/Aux.java
 1f2757c67215cf657297a009a01288108a04daeea919713871a2d34ea903b5f9  gencode/java/udmi/schema/BlobBlobsetConfig.java
 a7c57d119adcd0cf6363cc5301ce562004222522242e8ffd1d0cd7010f235ae1  gencode/java/udmi/schema/BlobBlobsetState.java
 b9f903444ab08907e41eb123286434ff3207b1edd01397af3ddefb8475bbdadc  gencode/java/udmi/schema/BlobsetConfig.java
 fcbed49f1af8b791d8c52bcbe18f65521a79d9ac3eb33ec3afd9b342ab2bfc56  gencode/java/udmi/schema/BlobsetState.java
-2f7cf83f8f46080c011eb4bebcb3b4a2125303f52d91def194a6818cdb944198  gencode/java/udmi/schema/Category.java
+0a4f6bcd5065418c1cdc6c05b900b3de31744847d25b6ab6de7aabb1e724710e  gencode/java/udmi/schema/BuildingTranslation.java
+5655b896f000b0108f438f9b04bfc092764ad5af63b70bbf750ba58b5f5527fa  gencode/java/udmi/schema/Category.java
 d6875f63ce67d1b945a0b75a4a660bd083cc52492371a7350c4109f6bf54968b  gencode/java/udmi/schema/CloudIotConfig.java
 a2eeff86f4302272736d84602e2ca36a64d27c8ef6761cc05ffb8ad17b030d4d  gencode/java/udmi/schema/CloudModel.java
 ff79de9390aa25bb45fb3e2ebb682c865ccab764f56d9644377d9d28c0ab10e4  gencode/java/udmi/schema/Config.java
+10d67bf2080403fd196f63097e4ce2151edaafe3cf4ac77598ef83e06f94cb05  gencode/java/udmi/schema/DeviceMappingConfig.java
+5ffb3bf92436c469eee16fe1e472efc89d1c466785be6ce30148188439314afe  gencode/java/udmi/schema/DeviceMappingState.java
 d4a82e132b2d223c4a82384d22455283c0b64aba3100fc210e72f9700b9df2d0  gencode/java/udmi/schema/DeviceValidationEvent.java
 587d67a67431349939dffd37b880c44e798a1eb607d54bd6d8a077bbac668067  gencode/java/udmi/schema/DiscoveryCommand.java
 d8a80ab3180d33bfa17564c969018e1d4350a47dbc70c4ae8a5abbfb25cfedc9  gencode/java/udmi/schema/DiscoveryConfig.java
@@ -28,9 +19,9 @@ d8a80ab3180d33bfa17564c969018e1d4350a47dbc70c4ae8a5abbfb25cfedc9  gencode/java/u
 9962b0eb7d5adf52af6160e9f3131f8eeb52ae9e518954dbb6aead1bcad0245e  gencode/java/udmi/schema/DiscoveryState.java
 f9faaa8622aa289c878cf2b9bcc05ac0c90ec5b40b39a607f5b7981fac0086fc  gencode/java/udmi/schema/EndpointConfiguration.java
 885859ea9de29217186c34f3a90b112411d1beddaf58a88b85e7934cd3c92b6f  gencode/java/udmi/schema/Entry.java
-cd362f94454eba8fd5ce3fce5d5e2b5f046d0dd9c35b01de69ef4d2e38413cc5  gencode/java/udmi/schema/Envelope.java
+9b92e03774ed0b9159ff3ab462822699ef7f824161b166a881889ea1eb6a62ad  gencode/java/udmi/schema/Envelope.java
 e9f5c77be81486b6b8c6d88f70f2d50583d8c3fafa2ac09ead80f44b8d5e751e  gencode/java/udmi/schema/Event.java
-aa0885ca43ab38c7597eacc38b7c512940a1a9fa061abd47d02c28e66b6fd93e  gencode/java/udmi/schema/FamilyDiscoveryConfig.java
+2e77a29988e6c17875d3f781be931e3b835bb310639f60d3bb28f24405a86bed  gencode/java/udmi/schema/FamilyDiscoveryConfig.java
 ae4a645f199c8e24b3303463d428ca17af7603ae9ae9238397a6a82e752ab454  gencode/java/udmi/schema/FamilyDiscoveryEvent.java
 0afc15acd72874e5a0c47f546abc0c4569f5bc37838fdcac77bc7bd55cc53a6d  gencode/java/udmi/schema/FamilyDiscoveryState.java
 9959a84eea3e549c142c3edf637c86eb56eca138108ebd51fc2985e45aa41484  gencode/java/udmi/schema/FamilyDiscoveryTestingModel.java
@@ -42,6 +33,10 @@ a5e5adfc187709e8646a11c92e804acfb67743f9d72149008aaca954df3177f6  gencode/java/u
 07fd4911363437b274c19b024759b04b116152176702da8d4203c4ff4cb55b7f  gencode/java/udmi/schema/LocalnetConfig.java
 910c68183db7703b00bcb81146ad73e6fe0d4bbc4caec4dc9c621f3cc2e5eee5  gencode/java/udmi/schema/LocalnetModel.java
 2df4ae32d0bbecc21f7c3f6a416a195baa766a6210cfa8abca4a7bb45b9c7961  gencode/java/udmi/schema/Location.java
+b815ba1f198fd32b11fbedf71e5fa820bba08713dc5c95603c700dfa21ec5904  gencode/java/udmi/schema/MappingCommand.java
+0f6f85a4aaffb90735acc2565e8e4afa349d3ded5781cb1dce1ebbaa237feaa7  gencode/java/udmi/schema/MappingConfig.java
+7f790aa98c03fcd00826a48fcf4dca7eda592abac994e5073c70be8dbd9b6745  gencode/java/udmi/schema/MappingEvent.java
+d2bf4eea0ca3df47b9ffe31481a52170e2d2bc3a0e7f2eab582e93cc20ccc886  gencode/java/udmi/schema/MappingState.java
 304164da05dc722b6e94cfa68659f0120e2425c94bfb5f5a4c6d796fe13da885  gencode/java/udmi/schema/Metadata.java
 a4e8f69100ab678a8236f481c558d677bbaea3e76c853bbd9262113d2a9c031d  gencode/java/udmi/schema/Metrics.java
 5e1c5411fae4d7c47391ceb5d19ae864fcd484df75ac6b6db39fd2d12647dec8  gencode/java/udmi/schema/Physical_tag.java
@@ -73,13 +68,17 @@ b2274107ce0e4e663b2a35e2ca284e492ec683bd687f6b8d8b014d5f8be67233  gencode/java/u
 7d6dd13e368e7f073738fee69c15e18652a9b7d7ac63bde0a200f747e3aa1b1d  gencode/java/udmi/schema/TargetTestingModel.java
 d3968b92497e83a63f18cc0e74484a9807f1bb92db0c92d556ec2caaa143d645  gencode/java/udmi/schema/TestingModel.java
 8a7dd590b84944de25392b1437184608edd2f8305a183eced8b1b40eae27c6d2  gencode/java/udmi/schema/TestingSystemConfig.java
-347da3d586ff6e5dcf3ce8ec0c800ec0efd5fed4e0c80eef35de65ad92ae48bf  gencode/java/udmi/schema/ValidationEvent.java
+ac6f8fd87c8986cce01e872460c15ff6fe71e3816f9bde610acfe25f7d38c8d4  gencode/java/udmi/schema/ValidationEvent.java
+f7d117dc8b9764acf0c95a13a2bfdfbdf31d1a8ec83a707448aa4d7391ef07e2  gencode/java/udmi/schema/ValidationState.java
 e007ddd1ceeae3603c85110c33e1bb4a418ff9c7a791ca0df25b7ea3caeafd36  gencode/java/udmi/schema/ValidationSummary.java
-1da7c14bbcbb4bd957259de271ed9585af915b3cebc4d3cf84b52a8f1b55d2a7  gencode/python/udmi/schema/__init__.py
+445723ac8a6c5e7072d344fbd865bbefce409bf9a07e194b6d82e9d0751547bd  gencode/python/udmi/schema/__init__.py
 4b25dd95f863059b761269f93adcae7049507924a1c6e74d6856849203c179db  gencode/python/udmi/schema/ancillary_properties.py
-30962f693f0065565f8f6610db934aa8ade58b7c9c978b6e0921d86336e3e1f2  gencode/python/udmi/schema/category.py
+5ecd6c542f33450cb4ce75d940a6dff4d3bd67d4b9de4aff5ee88abcc301dbff  gencode/python/udmi/schema/building_config.py
+dab4f5fca272ec48c2881bca2b6bc43786ada47fa1f6dd935c35f7ce0eb6b0f6  gencode/python/udmi/schema/building_translation.py
+d4269e665695f7a431108233a427313e609764f421d7657a23bd68d046f11e83  gencode/python/udmi/schema/category.py
 a61b1c5732f01b7efda41a773d5786fad755f371193ce4478b458387ca2a8fe8  gencode/python/udmi/schema/cloud_iot_config.py
 6578d68f65b87b781086e72566de910db4bef365599fe3188862d4d8a81e84fb  gencode/python/udmi/schema/command_discovery.py
+1254c34d973c9099ae99dcea4534e234e9019f49255e2e27d2afa1bc074fd596  gencode/python/udmi/schema/command_mapping.py
 c5a62f92328e2ede167fc1f53bece6e48696a0ab0e37e3a41f65cd98494ba0d7  gencode/python/udmi/schema/common.py
 b975892df78076dabc797b4c0be87f20b33eacda11f9d1ac1c09be33d4937a87  gencode/python/udmi/schema/config.py
 79eb0299b3751d93c01a5de65eacc717283e99caf0996f3db48a15247f69c8eb  gencode/python/udmi/schema/config_blobset.py
@@ -88,6 +87,8 @@ ec6c6ab1fb0f37a29b7ebd162aa77da7f1e261e80da376942a3b39d17ccf1be4  gencode/python
 a5edb9ac5ecd5a4459f93ce613691735f299f35718f2e35410206fc91c263dd1  gencode/python/udmi/schema/config_discovery_family.py
 b461bdc24310ef972faf579b5be577b5af67fb0977d6afb4c42955211b26e3d5  gencode/python/udmi/schema/config_gateway.py
 8b2bf3e753c09e375f0fd59dcf1f0be61205ed247160b7a8718cb3b9ebef2c30  gencode/python/udmi/schema/config_localnet.py
+ac3facbd96f7cb2f7e387e7497d6a36af379a2687329571f250c5670f9933244  gencode/python/udmi/schema/config_mapping.py
+1867ee8f1f4552ff121b7a2a62815817b2c9939a8817f1e093fbd688b8faa1a4  gencode/python/udmi/schema/config_mapping_device.py
 9eab64849e04b25203d5da47856c3f8dda2b96903e4dc43ab932ee35014700bd  gencode/python/udmi/schema/config_pointset.py
 607c5047df878a1333df3ce88dcce34668959b0b315f6954bf1a4963dcf7839e  gencode/python/udmi/schema/config_pointset_point.py
 ba37d2d54df565aba42ef055a0ec961175d180c2e8092a914e62029bc5388857  gencode/python/udmi/schema/config_system.py
@@ -99,11 +100,11 @@ ba37d2d54df565aba42ef055a0ec961175d180c2e8092a914e62029bc5388857  gencode/python
 82182e3f569ad80dc0751027959c7db9135d10072fbe79f896d63a4cd2f4771f  gencode/python/udmi/schema/event_discovery.py
 ad33b91a7fabb4eed7e49c30a983a2106c96330facbe0f376f94d06e2263d6d0  gencode/python/udmi/schema/event_discovery_family.py
 42e6fb6edda2b6aeae5e5b48ae66f87a5daf336bca1e746b1157e8f1ce31d5d5  gencode/python/udmi/schema/event_discovery_point.py
-abe4044d2e3be6693ed39edc8ccaed4eee4eb8acad991e820b21d6ecf3812dd9  gencode/python/udmi/schema/event_mapping.py
+5f41d6252d46f7b63e7a4cac731739b3e672f207eae2c65d448dfefc39b8a2ca  gencode/python/udmi/schema/event_mapping.py
 ddf849bfeb2b87d071cefd5e6feacabc57375a7fff6d72b6d42ffb89f33c859b  gencode/python/udmi/schema/event_pointset.py
 44aff1bc930dbdbadd51ac3fe0e7d9c83ad84a6a9f9d1c809b3fce66cbcd5e00  gencode/python/udmi/schema/event_pointset_point.py
 c3bf9959c821ccc8d0847a2e022e847ac3da8309dc6d35681af3d20148464ee4  gencode/python/udmi/schema/event_system.py
-24b7f3e103024c904e7eb8be52239d6a282221d5e080cf9c0be02a6aade4357d  gencode/python/udmi/schema/event_validation.py
+fb8a8a0e09eca4bb061c6cb2ff91b8eee77203e6ea353d3356e411692e1e4f2a  gencode/python/udmi/schema/event_validation.py
 2bb36986f805c2127f0c300de07bce07897c0c6d8f504bddec6e201d0f0ab1bf  gencode/python/udmi/schema/event_validation_device.py
 42f3429edf4a187be6ebfe9760f5e14821a25afa8f7c65b69e0e34fb29a7932e  gencode/python/udmi/schema/metadata.py
 b2227a48f21594f78c07c427de64e7ce51dc0ecd4b8091601d739a5f5038a2c9  gencode/python/udmi/schema/model_cloud.py
@@ -127,7 +128,10 @@ a58f8c98e837a5b56126ca0f410e02f1e9cfcd80a8cb429e0ef522defab1f690  gencode/python
 26443a1f6d0be3469ff93aa7fdb4e6682e0439a3b29a8e237998dcebec5f6901  gencode/python/udmi/schema/state_discovery.py
 187400078dfc89912062ca1ad92f61e32d28126ae56119d83e6767d58cda1117  gencode/python/udmi/schema/state_discovery_family.py
 05e82aa15c64842e206ae8ce3d5810d115bb890d009ea5d657822fad0e0d2165  gencode/python/udmi/schema/state_gateway.py
+cb6a5558fe87dea5bce9e1cc6b22984a7d8c379babc4941c50df13edad36579a  gencode/python/udmi/schema/state_mapping.py
+f2a76b32a5568d87a8c9ca8b32dc8a129bc7386f9522c5399d32db433be4cb70  gencode/python/udmi/schema/state_mapping_device.py
 3520ad936af70b414d9e7f90e606a011768bc4ee3bf1248714acc517ee9b393d  gencode/python/udmi/schema/state_pointset.py
 837ecc89c477abe3a1faf837733ca05475774891b55353d84ca231d90a1fbf31  gencode/python/udmi/schema/state_pointset_point.py
 8184e783f3e2da5a6d3fb7e7fac105aaeb28106a7c146e82b610049554abf184  gencode/python/udmi/schema/state_system.py
 791006619518fd7b38adb532879bcacd6f3f8795026cd75e10d3434a177757ac  gencode/python/udmi/schema/state_system_hardware.py
+231479e4dd7d961d59149580aba6a99cd466cea553400f338e9da2bf3ffbe78c  gencode/python/udmi/schema/state_validation.py

_includes/head-custom.html

@@ -0,0 +1,10 @@
+<script src="https://cdn.jsdelivr.net/npm/mermaid/dist/mermaid.min.js" defer="true"></script>
+<script>
+  document.addEventListener("DOMContentLoaded", function() {
+      document.querySelectorAll("pre>code.language-mermaid").forEach($element => {
+        $element.parentElement.outerHTML = `<div class="mermaid">${$element.textContent}</div>`
+      });
+     mermaid.initialize();
+});
+
+</script>

bin/cloud_logs

@@ -0,0 +1,14 @@
+#!/bin/bash -e
+
+project_id=$1
+
+if [[ -z $project_id ]]; then
+   echo $0 [project_id]
+   false
+fi
+
+for function in udmi_config udmi_target udmi_state udmi_reflect; do
+    echo Capturing $function.log...
+    gcloud --project=$project_id functions logs read $function --sort-by=time_utc --limit=1000 > $function.log
+done
+

bin/gencode

@@ -8,16 +8,21 @@ if [[ $1 == check ]]; then
     shift
 fi
 
+if [[ -n $check ]]; then
+    # This check must be run before gencode_docs is run
+    bin/gencode_docs_examples check
+fi
+
 bin/gencode_java
 bin/gencode_python gencode/python schema/*.json
 bin/gencode_docs
 bin/gencode_seq
 
 if [[ -n $check ]]; then
     echo Checking gencode docs links...
-    bin/gencode_docs check_links
+    bin/gencode_docs_checklinks
 
-    echo Checking gencode hash...
+    echo "Checking gencode hash (dynamic/static)..."
     files=`find gencode/ -type f | sort`
     sha256sum $files > /tmp/gencode.hash
     diff /tmp/gencode.hash .gencode_hash.txt

bin/gencode_categories

@@ -9,8 +9,10 @@ import sys
 
 GENCODE_MARKER = '@@ '
 CATEGORY_MARKER = '* '
+WILDCARD_REGEX = ' *\\* _\?\?\?_: (.*)'
 CATEGORY_REGEX = ' *\\* _([a-z]+)_: (\(\*\*([A-Z]+)\*\*\) )?(.*)'
 JSON_FORMAT = '%s%s{ "pattern": "^%s$" }'
+DEVICE_PREFIX = 'device['
 
 JAVA_DESCRIPTION = "\n%s// %s\n"
 JAVA_TARGET = '%spublic static final String %s = "%s";\n'
@@ -31,6 +33,11 @@ def read_categories():
     with open(doc_in) as doc:
         while line := doc.readline():
             indent = line.find(CATEGORY_MARKER)//2
+            wildcard = re.match(WILDCARD_REGEX, line)
+            if wildcard:
+                entry = (group + '[.a-z]+', 'INFO', wildcard.group(1))
+                categories.append(entry)
+                continue
             match = re.match(CATEGORY_REGEX, line)
             if indent < 0 or not match:
                 continue
@@ -83,6 +90,8 @@ def write_java_out(categories):
 def write_java_categories(out, indent, categories):
     for category in categories:
         target = category[0]
+        if target.startswith(DEVICE_PREFIX):
+            continue
         level = category[1]
         desc = category[2]
         const = target.replace('.', '_').upper()

bin/gencode_docs

@@ -28,12 +28,9 @@ fi
 OUTPUT_DIR=gencode/docs
 TMP_DIR=tmp/schema
 
-schema_files=`ls schema/*.json`
-
 if [ "$OP" == gen_bucket ]; then
-    rm -rf $OUTPUT_DIR $TMP_DIR
 
-    input_file=$OP
+    input_file=$1
     output_file=$2
     tmp_file=$output_file.tmp
 
@@ -47,7 +44,7 @@ if [ "$OP" == gen_bucket ]; then
     # Subheadings
     sed -E -i 's;^\t(\t*)([a-zA-Z0-9 .\-_\/]+)\t+$;\t\1*\t**\2**;g' $tmp_file
     # List Items
-    sed -E -i 's;^\t(\t*)([a-zA-Z0-9 .\-_\/]+)\t+([^\t\n].*)$;\t\1*\t`\2` - \3;g' $tmp_file
+    sed -E -i 's;^\t(\t*)([a-zA-Z0-9 .\-_\/]+)\t+([^\t\n].*)$;\t\1*\t_`\2`_ - \3;g' $tmp_file
     # Convert tabs to spaces (comply with style guide)
     expand -t 4 $tmp_file > $output_file
 
@@ -69,12 +66,12 @@ if [ "$OP" == check_links ]; then
 
     missing=
     echo Checking references to externally linked files...
-    fgrep -v \# etc/external_refs.txt | while read file comment; do
+    while read file comment; do
         if [[ -n $file && ! -f $file ]]; then
             echo "Reference to missing file '$file': $comment"
             missing=y
         fi
-    done
+    done < <( fgrep -v \# etc/external_refs.txt )
 
     if [[ -n $missing ]]; then
         exit 1
@@ -89,11 +86,10 @@ if [ -d venv ]; then
     source venv/bin/activate
 fi
 
-mkdir -p $OUTPUT_DIR
+rm -rf $OUTPUT_DIR $TMP_DIR
+mkdir -p $OUTPUT_DIR $TMP_DIR
 
 # Make a copy of the schema files into a tmp folder for any pre generator fixes
-rm -rf $TMP_DIR
-mkdir -p $TMP_DIR
 cp -r schema/. $TMP_DIR
 
 # Remove file: prefix from references so document generator works
@@ -103,7 +99,8 @@ find $TMP_DIR -type f -exec sed -i 's/file://' {} \;
 # is one level down
 find $TMP_DIR -type f -exec sed -i 's;(../docs/;(../../docs/;' {} \;
 
-schemas="state event_system event_pointset event_discovery event_validation config metadata envelope"
+schemas=$(bin/gencode_root_schemas schema/)
+echo Processing schemas $schemas
 
 for schema in $schemas; do
     echo == Processing $schema.json
@@ -112,15 +109,26 @@ done
 
 rm -rf $TMP_DIR
 
+echo Generating schema index page...
+
 # Create index page with links to different schemas
-cp etc/schema_index_template.html $OUTPUT_DIR/index.html
+cp etc/schema_readme_template.md $OUTPUT_DIR/readme.md
 
 for schema in $schemas; do
-    sed -i "s/<!--SCHEMALINK-->/\
-        <div class=\"card\"><a href=\"$schema.html\">$schema<\/a><\/div><!--SCHEMALINK-->/"\
-        $OUTPUT_DIR/index.html
+    section=$(jq -r '.["$section"]' schema/$schema.json)
+    if [[ -z $section || $(grep -c "## $section\$" $OUTPUT_DIR/readme.md) -eq 0 ]]; then
+        section=Other
+    fi
+    description=$(jq -r .description schema/$schema.json)
+    if [[ $description == null ]]; then
+        description=""
+    else
+        description=$(echo " - $description" | sed 's@(\.\./@(\.\./\.\./@g')
+    fi
+    sed -i "s@## $section@## $section\n* [**$schema**]($schema.html)$description@" $OUTPUT_DIR/readme.md
 done
 
+
 # Add navbar
 navbar=$(cat etc/schema_navbar.html)
 navbar=$(printf '%q ' $navbar)
@@ -133,3 +141,9 @@ find $OUTPUT_DIR -type f -exec sed -i -E "s;</head>;<script src=\"schema_extras.
 
 find $OUTPUT_DIR -type f -exec sed -i -E "s/<body onload=\"anchorOnLoad\(\);\"/\
     <body onload=\"anchorOnLoad\(\);removedHostedMDExtensions\(\);\"/" {} \;
+
+
+# Update inline documentation examples
+bin/gencode_docs_examples
+
+echo Done with $0