From 4b3824a2b026e6337984800f562b041ba0d5e182 Mon Sep 17 00:00:00 2001 From: John Randolph Date: Thu, 2 Jun 2022 12:52:38 -0400 Subject: [PATCH 01/57] Add dryrun option -n --- bin/clean_tags | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/bin/clean_tags b/bin/clean_tags index 20ee36310c..30b57285c9 100755 --- a/bin/clean_tags +++ b/bin/clean_tags @@ -1,5 +1,15 @@ #!/bin/bash -e +dryrun="" + +while getopts "n" options; do + case "${options}" in + n) + dryrun="echo" + ;; + esac +done + if [[ -n $1 ]]; then repo=$1 shift @@ -10,9 +20,9 @@ for tag in $(git tag); do # A bad tag is one that does not have a common parent with master branch. merge=$(git merge-base $tag master || true) if [[ -z $merge ]]; then - git tag -d $tag + ${dryrun} git tag -d $tag if [[ -n $repo ]]; then - git push $repo :$tag + ${dryrun} git push $repo :$tag fi fi done From e0c6de0045eed780d18482c89e6e127ed9f69b69 Mon Sep 17 00:00:00 2001 From: John Randolph Date: Thu, 2 Jun 2022 13:22:07 -0400 Subject: [PATCH 02/57] Oops, shift by the option --- bin/clean_tags | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bin/clean_tags b/bin/clean_tags index 30b57285c9..19f578e198 100755 --- a/bin/clean_tags +++ b/bin/clean_tags @@ -10,12 +10,17 @@ while getopts "n" options; do esac done +shift $((OPTIND-1)) + if [[ -n $1 ]]; then repo=$1 shift git fetch $repo --tags fi +echo repo = $repo +exit 1 + for tag in $(git tag); do # A bad tag is one that does not have a common parent with master branch. merge=$(git merge-base $tag master || true) From a6b72b226d2f6f761904ef5591346d2e055772fc Mon Sep 17 00:00:00 2001 From: John Randolph Date: Thu, 2 Jun 2022 13:22:48 -0400 Subject: [PATCH 03/57] Remove dbg --- bin/clean_tags | 3 --- 1 file changed, 3 deletions(-) diff --git a/bin/clean_tags b/bin/clean_tags index 19f578e198..23faf428b4 100755 --- a/bin/clean_tags +++ b/bin/clean_tags @@ -18,9 +18,6 @@ if [[ -n $1 ]]; then git fetch $repo --tags fi -echo repo = $repo -exit 1 - for tag in $(git tag); do # A bad tag is one that does not have a common parent with master branch. merge=$(git merge-base $tag master || true) From b60ecdb2e4e352de726bd846d06b845e5666348b Mon Sep 17 00:00:00 2001 From: John Randolph Date: Thu, 9 Jun 2022 10:55:49 -0400 Subject: [PATCH 04/57] First pass at RegistrarTest and slight changes to Registrar. --- .../google/daq/mqtt/registrar/Registrar.java | 8 +- .../daq/mqtt/registrar/RegistrarTest.java | 88 +++++++++++++++++++ 2 files changed, 92 insertions(+), 4 deletions(-) create mode 100644 validator/src/main/java/com/google/daq/mqtt/registrar/RegistrarTest.java diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java b/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java index 1bcca64645..0b1ad08da7 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java @@ -81,7 +81,7 @@ public class Registrar { private static final int RUNNER_THREADS = 25; private static final String CONFIG_SUB_TYPE = "config"; private static final String MODEL_SUB_TYPE = "model"; - private final Map schemas = new HashMap<>(); + protected final Map schemas = new HashMap<>(); private final String generation = getGenerationString(); private CloudIotManager cloudIotManager; private File siteDir; @@ -218,7 +218,7 @@ private void writeErrors() throws Exception { OBJECT_MAPPER.writeValue(summaryFile, errorSummary); } - private void setSitePath(String sitePath) { + protected void setSitePath(String sitePath) { Preconditions.checkNotNull(SCHEMA_NAME, "schemaName not set yet"); siteDir = new File(sitePath); summaryFile = new File(siteDir, REGISTRATION_SUMMARY_JSON); @@ -645,7 +645,7 @@ private Map loadDevices(File siteDir, File devicesDir, return localDevices; } - private void setProjectId(String projectId) { + protected void setProjectId(String projectId) { if (NO_SITE.equals(projectId) || projectId == null) { return; } @@ -653,7 +653,7 @@ private void setProjectId(String projectId) { initializeCloudProject(); } - private void setToolRoot(String toolRoot) { + protected void setToolRoot(String toolRoot) { schemaBase = new File(toolRoot, SCHEMA_BASE_PATH); File[] schemaFiles = schemaBase.listFiles(file -> file.getName().endsWith(SCHEMA_SUFFIX)); for (File schemaFile : Objects.requireNonNull(schemaFiles)) { diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/RegistrarTest.java b/validator/src/main/java/com/google/daq/mqtt/registrar/RegistrarTest.java new file mode 100644 index 0000000000..40166fc6b0 --- /dev/null +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/RegistrarTest.java @@ -0,0 +1,88 @@ +package com.google.daq.mqtt.registrar; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.util.ISO8601DateFormat; +import com.github.fge.jsonschema.core.exceptions.ProcessingException; +import com.github.fge.jsonschema.core.load.configuration.LoadingConfiguration; +import com.github.fge.jsonschema.main.JsonSchema; +import com.github.fge.jsonschema.main.JsonSchemaFactory; +import com.google.daq.mqtt.registrar.Registrar.RelativeDownloader; +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; +import java.util.List; +import org.junit.Test; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import udmi.schema.Asset; +import udmi.schema.Location; +import udmi.schema.Metadata; +import udmi.schema.Physical_tag; +import udmi.schema.SystemModel; + +public class RegistrarTest { + + public static final String SCHEMA_BASE_PATH = "schema"; + private static final String METADATA_JSON = "metadata.json"; + private static final String PROJECT_ID = "unit-testing"; + private static final String SITE_PATH = "../sites/udmi_site_model"; + private static final String TOOL_ROOT = "../"; + + private static final String SYSTEM_LOCATION_SITE = "ZZ-TRI-FECTA"; + private static final String DEVICE_NAME = "AHU-1"; + private ObjectMapper mapper = new ObjectMapper(); + + public class RegistrarUnderTest extends Registrar { + protected JsonSchema getJsonSchema(String schemaName) { + return schemas.get(schemaName); + } + } + + @Test public void emptyTest() { + Registrar registrar = getRegistrarUnderTest(); + assertEquals(1, 1); + } + + @Test public void metadataTest() throws JsonProcessingException, ProcessingException { + RegistrarUnderTest registrar = getRegistrarUnderTest(); + String metadata = getTestMetadataAsString(); + JsonSchema validator = registrar.getJsonSchema(METADATA_JSON); + validator.validate(mapper.readTree(metadata)); + } + + private RegistrarUnderTest getRegistrarUnderTest() { + RegistrarUnderTest registrar = new RegistrarUnderTest(); + registrar.setSitePath(SITE_PATH); + registrar.setProjectId(PROJECT_ID); + registrar.setToolRoot(TOOL_ROOT); + return registrar; + } + + private Metadata getTestMetadata() { + Metadata metadata = new Metadata(); + metadata.system = new SystemModel(); + /* + metadata.system.location = new Location(); + metadata.system.location.site = SYSTEM_LOCATION_SITE; + */ + metadata.system.physical_tag = new Physical_tag(); + metadata.system.physical_tag.asset = new Asset(); + metadata.system.physical_tag.asset.name = DEVICE_NAME; + metadata.description = "Test Metadata"; + return metadata; + } + + private String getTestMetadataAsString() throws JsonProcessingException { + return mapper.writeValueAsString(getTestMetadata()); + } + +} From 2ad6e879daa876066438719aed23150a1c15e183 Mon Sep 17 00:00:00 2001 From: John Randolph Date: Tue, 14 Jun 2022 11:33:48 -0400 Subject: [PATCH 05/57] Add validation tests --- .../google/daq/mqtt/registrar/Registrar.java | 6 +- .../daq/mqtt/registrar/RegistrarTest.java | 88 ++++++++++--------- .../daq/mqtt/validator/ValidatorTest.java | 9 ++ 3 files changed, 60 insertions(+), 43 deletions(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java b/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java index 0b1ad08da7..750c2a5ba3 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java @@ -81,7 +81,7 @@ public class Registrar { private static final int RUNNER_THREADS = 25; private static final String CONFIG_SUB_TYPE = "config"; private static final String MODEL_SUB_TYPE = "model"; - protected final Map schemas = new HashMap<>(); + private final Map schemas = new HashMap<>(); private final String generation = getGenerationString(); private CloudIotManager cloudIotManager; private File siteDir; @@ -706,6 +706,10 @@ private void loadSiteMetadata() { } } + protected Map getSchemas() { + return schemas; + } + class RelativeDownloader implements URIDownloader { @Override diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/RegistrarTest.java b/validator/src/main/java/com/google/daq/mqtt/registrar/RegistrarTest.java index 40166fc6b0..afcddc43bb 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/RegistrarTest.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/RegistrarTest.java @@ -2,32 +2,23 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import com.fasterxml.jackson.annotation.JsonInclude.Include; +import static org.junit.Assert.fail; + import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; -import com.fasterxml.jackson.databind.util.ISO8601DateFormat; import com.github.fge.jsonschema.core.exceptions.ProcessingException; -import com.github.fge.jsonschema.core.load.configuration.LoadingConfiguration; +import com.github.fge.jsonschema.core.report.LogLevel; +import com.github.fge.jsonschema.core.report.ProcessingMessage; +import com.github.fge.jsonschema.core.report.ProcessingReport; import com.github.fge.jsonschema.main.JsonSchema; -import com.github.fge.jsonschema.main.JsonSchemaFactory; -import com.google.daq.mqtt.registrar.Registrar.RelativeDownloader; import java.io.File; import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; import java.io.InputStream; -import java.util.List; import org.junit.Test; -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; -import udmi.schema.Asset; -import udmi.schema.Location; import udmi.schema.Metadata; -import udmi.schema.Physical_tag; -import udmi.schema.SystemModel; public class RegistrarTest { @@ -43,20 +34,51 @@ public class RegistrarTest { public class RegistrarUnderTest extends Registrar { protected JsonSchema getJsonSchema(String schemaName) { - return schemas.get(schemaName); + return getSchemas().get(schemaName); } } - @Test public void emptyTest() { - Registrar registrar = getRegistrarUnderTest(); - assertEquals(1, 1); + private InputStream getTestFileStream(String filename) throws FileNotFoundException { + File f = new File("/home/jrand/src/johnrandolph/udmi/tests/metadata.tests/", filename); + return new FileInputStream(f); + } + + private Metadata getTestMetadataValue() throws IOException { + return mapper.readValue(getTestFileStream("example.json"), Metadata.class); + } + + private JsonNode getTestMetadataTree() throws IOException { + return mapper.readTree(getTestFileStream("example.json")); } - @Test public void metadataTest() throws JsonProcessingException, ProcessingException { + private JsonNode getMetadataAsJsonNode(Metadata metadata) throws JsonProcessingException { + return mapper.readTree(mapper.writeValueAsString(metadata)); + } + + private void assertSuccessReport(ProcessingReport report) { + if (!report.isSuccess()) { + for (ProcessingMessage msg : report) { + if (msg.getLogLevel().compareTo(LogLevel.ERROR) >= 0) { + int i = 0; + fail(msg.getMessage().toString()); + } + } + } + } + + @Test public void metadataTest() throws IOException, ProcessingException, FileNotFoundException { RegistrarUnderTest registrar = getRegistrarUnderTest(); - String metadata = getTestMetadataAsString(); JsonSchema validator = registrar.getJsonSchema(METADATA_JSON); - validator.validate(mapper.readTree(metadata)); + + ProcessingReport report = registrar.getSchemas().get(METADATA_JSON).validate(getTestMetadataTree()); + assertSuccessReport(report); + + Metadata metadata = getTestMetadataValue(); + metadata.system = null; + JsonNode n = getMetadataAsJsonNode(metadata); + + report = registrar.getSchemas().get(METADATA_JSON).validate(n); + assertSuccessReport(report); } private RegistrarUnderTest getRegistrarUnderTest() { @@ -67,22 +89,4 @@ private RegistrarUnderTest getRegistrarUnderTest() { return registrar; } - private Metadata getTestMetadata() { - Metadata metadata = new Metadata(); - metadata.system = new SystemModel(); - /* - metadata.system.location = new Location(); - metadata.system.location.site = SYSTEM_LOCATION_SITE; - */ - metadata.system.physical_tag = new Physical_tag(); - metadata.system.physical_tag.asset = new Asset(); - metadata.system.physical_tag.asset.name = DEVICE_NAME; - metadata.description = "Test Metadata"; - return metadata; - } - - private String getTestMetadataAsString() throws JsonProcessingException { - return mapper.writeValueAsString(getTestMetadata()); - } - } diff --git a/validator/src/test/java/com/google/daq/mqtt/validator/ValidatorTest.java b/validator/src/test/java/com/google/daq/mqtt/validator/ValidatorTest.java index 12ecc160d7..bde81f385e 100644 --- a/validator/src/test/java/com/google/daq/mqtt/validator/ValidatorTest.java +++ b/validator/src/test/java/com/google/daq/mqtt/validator/ValidatorTest.java @@ -50,6 +50,15 @@ public class ValidatorTest { validator.setSiteDir(SITE_DIR); } + @Test + public void emptySystemBlock() { + MessageBundle bundle = getMessageBundle("event", "pointset", new PointsetEvent()); + bundle.message.remove("system"); + validator.validateMessage(bundle); + MetadataReport report = getMetadataReport(); + assertEquals("One error device", 1, report.errorDevices.size()); + } + @Test public void emptyPointsetEvent() { MessageBundle bundle = getMessageBundle("event", "pointset", new PointsetEvent()); From d15976824e12190a2799ea84b974df49069f4955 Mon Sep 17 00:00:00 2001 From: John Randolph Date: Tue, 21 Jun 2022 17:47:09 -0400 Subject: [PATCH 06/57] Test work --- .../daq/mqtt/registrar/LocalDevice.java | 28 +++++++++- .../google/daq/mqtt/registrar/Registrar.java | 15 ++++++ .../daq/mqtt/registrar/RegistrarTest.java | 51 +++++++++++++------ 3 files changed, 77 insertions(+), 17 deletions(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java b/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java index 3dc0c2bbf5..57f0748d81 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java @@ -17,6 +17,8 @@ import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.databind.util.ISO8601DateFormat; import com.github.fge.jsonschema.core.exceptions.ProcessingException; +import com.github.fge.jsonschema.core.report.LogLevel; +import com.github.fge.jsonschema.core.report.ProcessingMessage; import com.github.fge.jsonschema.core.report.ProcessingReport; import com.github.fge.jsonschema.main.JsonSchema; import com.google.api.services.cloudiot.v1.model.DeviceCredential; @@ -198,6 +200,7 @@ class LocalDevice { private final String generation; private final List deviceCredentials = new ArrayList<>(); private final TreeMap siteMetadata; + private final Boolean validateMetadata; private String deviceNumId; @@ -211,6 +214,7 @@ class LocalDevice { this.schemas = schemas; this.generation = generation; this.siteDir = siteDir; + this.validateMetadata = true; if (siteMetadata != null) { this.siteMetadata = OBJECT_MAPPER.convertValue(siteMetadata, TreeMap.class); } else { @@ -232,6 +236,18 @@ class LocalDevice { this(siteDir, devicesDir, deviceId, schemas, generation, null); } + public static void parseMetadataValidateProcessingReport(ProcessingReport report) throws ValidationException { + if (report.isSuccess()) { + return; + } + + for (ProcessingMessage msg : report) { + if (msg.getLogLevel().compareTo(LogLevel.ERROR) >= 0) { + throw ValidationException.fromProcessingReport(report); + } + } + } + private void prepareOutDir() { if (!outDir.exists()) { outDir.mkdir(); @@ -273,6 +289,7 @@ public void validateExpected() { exceptionMap.throwIfNotEmpty(); } + private void deepMergeDefaults(Map destination, Map source) { for (String key : source.keySet()) { Object value2 = source.get(key); @@ -289,10 +306,13 @@ private void deepMergeDefaults(Map destination, Map cloudDevices; private Metadata siteMetadata; + private Map> lastErrorSummary; public static void main(String[] args) { ArrayList argList = new ArrayList<>(List.of(args)); Registrar registrar = new Registrar(); + executeWithRegistrar(registrar, argList); + } + + public static void executeWithRegistrar(Registrar registrar, ArrayList argList) { try { boolean processAllDevices = processArgs(argList, registrar); @@ -179,6 +186,10 @@ private void setFeedTopic(String feedTopic) { } } + protected Map> getLastErrorSummary() { + return lastErrorSummary; + } + private void writeErrors() throws Exception { Map> errorSummary = new TreeMap<>(); DeviceExceptionManager dem = new DeviceExceptionManager(siteDir); @@ -216,6 +227,7 @@ private void writeErrors() throws Exception { String version = Optional.ofNullable(System.getenv(UDMI_VERSION_KEY)).orElse("unknown"); errorSummary.put(VERSION_KEY, Map.of(VERSION_MAIN_KEY, version)); OBJECT_MAPPER.writeValue(summaryFile, errorSummary); + lastErrorSummary = errorSummary; } protected void setSitePath(String sitePath) { @@ -691,6 +703,9 @@ private void loadSiteMetadata() { File siteMetadataFile = new File(siteDir, SITE_METADATA_JSON); try (InputStream targetStream = new FileInputStream(siteMetadataFile)) { + // At this time, do not validate the Metadata schema because, by its nature of being + // a partial overlay on each device Metadata, this Metadata will likely be incomplete + // and fail validation. schemas.get(METADATA_JSON).validate(OBJECT_MAPPER.readTree(targetStream)); } catch (FileNotFoundException e) { return; diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/RegistrarTest.java b/validator/src/main/java/com/google/daq/mqtt/registrar/RegistrarTest.java index afcddc43bb..30aa780da8 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/RegistrarTest.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/RegistrarTest.java @@ -17,6 +17,9 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.util.ArrayList; +import java.util.Map; +import java.util.TreeMap; import org.junit.Test; import udmi.schema.Metadata; @@ -38,17 +41,19 @@ protected JsonSchema getJsonSchema(String schemaName) { } } + + /* private InputStream getTestFileStream(String filename) throws FileNotFoundException { File f = new File("/home/jrand/src/johnrandolph/udmi/tests/metadata.tests/", filename); return new FileInputStream(f); } - private Metadata getTestMetadataValue() throws IOException { - return mapper.readValue(getTestFileStream("example.json"), Metadata.class); + private Metadata getTestMetadataValue(String filename) throws IOException { + return mapper.readValue(getTestFileStream(filename), Metadata.class); } - private JsonNode getTestMetadataTree() throws IOException { - return mapper.readTree(getTestFileStream("example.json")); + private JsonNode getTestMetadataTree(String filename) throws IOException { + return mapper.readTree(getTestFileStream(filename)); } private JsonNode getMetadataAsJsonNode(Metadata metadata) throws JsonProcessingException { @@ -66,19 +71,35 @@ private void assertSuccessReport(ProcessingReport report) { } } - @Test public void metadataTest() throws IOException, ProcessingException, FileNotFoundException { - RegistrarUnderTest registrar = getRegistrarUnderTest(); - JsonSchema validator = registrar.getJsonSchema(METADATA_JSON); - - ProcessingReport report = registrar.getSchemas().get(METADATA_JSON).validate(getTestMetadataTree()); - assertSuccessReport(report); + private void assertFailReport(ProcessingReport report, String message) { + if (!report.isSuccess()) { + for (ProcessingMessage msg : report) { + if (msg.getLogLevel().compareTo(LogLevel.ERROR) >= 0) { + if (msg.getMessage().contains(message)) { + return; + } + fail(msg.getMessage().toString()); + } + } + } + fail("ProcessingReport had no errors or no matching message"); + } + */ - Metadata metadata = getTestMetadataValue(); - metadata.system = null; - JsonNode n = getMetadataAsJsonNode(metadata); + private void assertErrorSummarySuccess(Map> summary) { + if (summary.get("Validating").size() == 0) { + return; + } + fail(summary.get("Validating").toString()); + } - report = registrar.getSchemas().get(METADATA_JSON).validate(n); - assertSuccessReport(report); + @Test public void metadataValidateSuccessTest() { + RegistrarUnderTest registrar = getRegistrarUnderTest(); + ArrayList argList = new ArrayList(); + argList.add("-s"); + argList.add("/home/jrand/src/johnrandolph/udmi_site_model"); + registrar.executeWithRegistrar(registrar, argList); + assertErrorSummarySuccess(registrar.getLastErrorSummary()); } private RegistrarUnderTest getRegistrarUnderTest() { From 6793effc3c43e9520c42656130924a006494a2aa Mon Sep 17 00:00:00 2001 From: John Randolph Date: Wed, 22 Jun 2022 12:46:04 -0400 Subject: [PATCH 07/57] Test improvements --- .../daq/mqtt/registrar/LocalDevice.java | 14 +++- .../google/daq/mqtt/registrar/Registrar.java | 10 ++- .../daq/mqtt/registrar/RegistrarTest.java | 84 ++++++++----------- 3 files changed, 52 insertions(+), 56 deletions(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java b/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java index 57f0748d81..f5a16c14e4 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java @@ -200,7 +200,7 @@ class LocalDevice { private final String generation; private final List deviceCredentials = new ArrayList<>(); private final TreeMap siteMetadata; - private final Boolean validateMetadata; + private final boolean validateMetadata; private String deviceNumId; @@ -208,13 +208,13 @@ class LocalDevice { LocalDevice( File siteDir, File devicesDir, String deviceId, Map schemas, - String generation, Metadata siteMetadata) { + String generation, Metadata siteMetadata, boolean validateMetadata) { try { this.deviceId = deviceId; this.schemas = schemas; this.generation = generation; this.siteDir = siteDir; - this.validateMetadata = true; + this.validateMetadata = validateMetadata; if (siteMetadata != null) { this.siteMetadata = OBJECT_MAPPER.convertValue(siteMetadata, TreeMap.class); } else { @@ -230,6 +230,12 @@ class LocalDevice { } } + LocalDevice( + File siteDir, File devicesDir, String deviceId, Map schemas, + String generation, Metadata siteMetadata) { + this(siteDir, devicesDir, deviceId, schemas, generation, siteMetadata, false); + } + LocalDevice( File siteDir, File devicesDir, String deviceId, Map schemas, String generation) { @@ -306,7 +312,7 @@ private void deepMergeDefaults(Map destination, Map cloudDevices; private Metadata siteMetadata; private Map> lastErrorSummary; + private boolean validateMetadata = false; public static void main(String[] args) { ArrayList argList = new ArrayList<>(List.of(args)); @@ -155,6 +156,9 @@ private static boolean processArgs(List argList, Registrar registrar) { case "-l": registrar.setIdleLimit(argList.remove(0)); break; + case "-t": + registrar.setValidateMetadata(true); + break; case "--": return false; default: @@ -177,6 +181,10 @@ private void setUpdateFlag(boolean update) { updateCloudIoT = update; } + private void setValidateMetadata(boolean validateMetadata) { + this.validateMetadata = validateMetadata; + } + private void setFeedTopic(String feedTopic) { System.err.println("Sending device feed to topic " + feedTopic); feedPusher = new PubSubPusher(projectId, feedTopic); @@ -636,7 +644,7 @@ private Map loadDevices(File siteDir, File devicesDir, localDevices.computeIfAbsent( deviceName, keyName -> new LocalDevice(siteDir, devicesDir, deviceName, schemas, generation, - siteMetadata)); + siteMetadata, validateMetadata)); try { localDevice.loadCredentials(); } catch (Exception e) { diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/RegistrarTest.java b/validator/src/main/java/com/google/daq/mqtt/registrar/RegistrarTest.java index 30aa780da8..d3d6fdc712 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/RegistrarTest.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/RegistrarTest.java @@ -41,65 +41,26 @@ protected JsonSchema getJsonSchema(String schemaName) { } } - - /* - private InputStream getTestFileStream(String filename) throws FileNotFoundException { - File f = new File("/home/jrand/src/johnrandolph/udmi/tests/metadata.tests/", filename); - return new FileInputStream(f); - } - - private Metadata getTestMetadataValue(String filename) throws IOException { - return mapper.readValue(getTestFileStream(filename), Metadata.class); - } - - private JsonNode getTestMetadataTree(String filename) throws IOException { - return mapper.readTree(getTestFileStream(filename)); - } - - private JsonNode getMetadataAsJsonNode(Metadata metadata) throws JsonProcessingException { - return mapper.readTree(mapper.writeValueAsString(metadata)); - } - - private void assertSuccessReport(ProcessingReport report) { - if (!report.isSuccess()) { - for (ProcessingMessage msg : report) { - if (msg.getLogLevel().compareTo(LogLevel.ERROR) >= 0) { - int i = 0; - fail(msg.getMessage().toString()); - } - } + private void assertErrorSummaryValidateSuccess(Map> summary) { + if (summary == null) { + return; } - } - - private void assertFailReport(ProcessingReport report, String message) { - if (!report.isSuccess()) { - for (ProcessingMessage msg : report) { - if (msg.getLogLevel().compareTo(LogLevel.ERROR) >= 0) { - if (msg.getMessage().contains(message)) { - return; - } - fail(msg.getMessage().toString()); - } - } + if (summary.get("Validating") == null) { + return; } - fail("ProcessingReport had no errors or no matching message"); - } - */ - - private void assertErrorSummarySuccess(Map> summary) { if (summary.get("Validating").size() == 0) { return; } fail(summary.get("Validating").toString()); } - @Test public void metadataValidateSuccessTest() { - RegistrarUnderTest registrar = getRegistrarUnderTest(); - ArrayList argList = new ArrayList(); - argList.add("-s"); - argList.add("/home/jrand/src/johnrandolph/udmi_site_model"); - registrar.executeWithRegistrar(registrar, argList); - assertErrorSummarySuccess(registrar.getLastErrorSummary()); + private void assertErrorSummaryValidateFailure(Map> summary) { + if ((summary == null) || (summary.get("Validating") == null)) { + fail("Error summary for Validating key is null"); + } + if (summary.get("Validating").size()==0) { + fail("Error summary for Validating key is size 0"); + } } private RegistrarUnderTest getRegistrarUnderTest() { @@ -110,4 +71,25 @@ private RegistrarUnderTest getRegistrarUnderTest() { return registrar; } + @Test public void metadataValidateSuccessTest() { + RegistrarUnderTest registrar = getRegistrarUnderTest(); + + ArrayList argList = new ArrayList(); + argList.add("-s"); + argList.add(SITE_PATH); + registrar.executeWithRegistrar(registrar, argList); + assertErrorSummaryValidateSuccess(registrar.getLastErrorSummary()); + } + + @Test public void metadataValidateFailureTest() { + RegistrarUnderTest registrar = getRegistrarUnderTest(); + + ArrayList argList = new ArrayList(); + argList.add("-t"); + argList.add("-s"); + argList.add(SITE_PATH); + registrar.executeWithRegistrar(registrar, argList); + assertErrorSummaryValidateFailure(registrar.getLastErrorSummary()); + } + } From 167e8de5a6953479075afa262f03631065ed4292 Mon Sep 17 00:00:00 2001 From: John Randolph Date: Wed, 22 Jun 2022 15:04:52 -0400 Subject: [PATCH 08/57] Merge --- .../com/google/daq/mqtt/registrar/LocalDevice.java | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java b/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java index f87f1bee4c..3ff2add193 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java @@ -311,19 +311,16 @@ private Metadata readMetadataWithValidation(boolean validate) { instance = OBJECT_MAPPER.readTree(targetStream); baseVersion = instance.get("version"); new MessageUpgrader("metadata", instance).upgrade(); - ProcessingReport report = schemas.get(METADATA_JSON).validate(instance); - if (validate) { - parseMetadataValidateProcessingReport(report); - } - } catch (ProcessingException | ValidationException e) { - exceptionMap.put(EXCEPTION_VALIDATING, e); } catch (IOException ioException) { exceptionMap.put(EXCEPTION_LOADING, ioException); return null; } try { - schemas.get(METADATA_JSON).validate(instance); + ProcessingReport report = schemas.get(METADATA_JSON).validate(instance); + if (validate) { + parseMetadataValidateProcessingReport(report); + } } catch (ProcessingException | ValidationException e) { exceptionMap.put(EXCEPTION_VALIDATING, e); } From eeb32b41aadf4a265933af6d0e4b299a810ba4a0 Mon Sep 17 00:00:00 2001 From: John Randolph Date: Wed, 22 Jun 2022 17:28:02 -0400 Subject: [PATCH 09/57] Fixes from first review --- .../com/google/daq/mqtt/registrar/LocalDevice.java | 1 - .../com/google/daq/mqtt/registrar/RegistrarTest.java | 11 +++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java b/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java index 3ff2add193..d2b6df87da 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java @@ -287,7 +287,6 @@ public void validateExpected() { exceptionMap.throwIfNotEmpty(); } - private void deepMergeDefaults(Map destination, Map source) { for (String key : source.keySet()) { Object value2 = source.get(key); diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/RegistrarTest.java b/validator/src/main/java/com/google/daq/mqtt/registrar/RegistrarTest.java index d3d6fdc712..d9b331de04 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/RegistrarTest.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/RegistrarTest.java @@ -25,7 +25,7 @@ public class RegistrarTest { - public static final String SCHEMA_BASE_PATH = "schema"; + private static final String SCHEMA_BASE_PATH = "schema"; private static final String METADATA_JSON = "metadata.json"; private static final String PROJECT_ID = "unit-testing"; private static final String SITE_PATH = "../sites/udmi_site_model"; @@ -42,13 +42,8 @@ protected JsonSchema getJsonSchema(String schemaName) { } private void assertErrorSummaryValidateSuccess(Map> summary) { - if (summary == null) { - return; - } - if (summary.get("Validating") == null) { - return; - } - if (summary.get("Validating").size() == 0) { + if ((summary == null) || (summary.get("Validating") == null) || + (summary.get("Validating").size() == 0)) { return; } fail(summary.get("Validating").toString()); From cc2f8b687553531bf6812db1e9f5142ee7c80093 Mon Sep 17 00:00:00 2001 From: John Randolph Date: Thu, 23 Jun 2022 11:24:45 -0400 Subject: [PATCH 10/57] Move execute to method, keep argv parsing in class --- .../google/daq/mqtt/registrar/Registrar.java | 64 +++++++++---------- .../daq/mqtt/registrar/RegistrarTest.java | 6 +- 2 files changed, 35 insertions(+), 35 deletions(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java b/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java index 4dc0bdc1b1..5932fbce9a 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java @@ -102,6 +102,7 @@ public class Registrar { private Metadata siteMetadata; private Map> lastErrorSummary; private boolean validateMetadata = false; + private List deviceList; /** * Main entry point for registrar. @@ -111,37 +112,11 @@ public class Registrar { public static void main(String[] args) { ArrayList argList = new ArrayList<>(List.of(args)); Registrar registrar = new Registrar(); - executeWithRegistrar(registrar, argList); + processArgs(argList, registrar); + registrar.execute(); } - public static void executeWithRegistrar(Registrar registrar, ArrayList argList) { - try { - boolean processAllDevices = processArgs(argList, registrar); - - if (registrar.schemaBase == null) { - registrar.setToolRoot(null); - } - - registrar.loadSiteMetadata(); - - if (processAllDevices) { - registrar.processDevices(); - } else { - registrar.processDevices(argList); - } - - registrar.writeErrors(); - registrar.shutdown(); - } catch (ExceptionMap em) { - ExceptionMap.format(em, ERROR_FORMAT_INDENT).write(System.err); - throw new RuntimeException("mapped exceptions", em); - } catch (Exception ex) { - ex.printStackTrace(); - throw new RuntimeException("main exception", ex); - } - } - - private static boolean processArgs(List argList, Registrar registrar) { + public static void processArgs(List argList, Registrar registrar) { while (argList.size() > 0) { String option = argList.remove(0); switch (option) { @@ -167,16 +142,35 @@ private static boolean processArgs(List argList, Registrar registrar) { registrar.setValidateMetadata(true); break; case "--": - return false; + break; default: if (option.startsWith("-")) { throw new RuntimeException("Unknown cmdline option " + option); } + // Add the current non-option back into the list and use it as device names list. argList.add(0, option); - return false; + registrar.setDeviceList(argList); + return; } } - return true; + } + + public void execute() { + try { + if (schemaBase == null) { + setToolRoot(null); + } + loadSiteMetadata(); + processDevices(); + writeErrors(); + shutdown(); + } catch (ExceptionMap em) { + ExceptionMap.format(em, ERROR_FORMAT_INDENT).write(System.err); + throw new RuntimeException("mapped exceptions", em); + } catch (Exception ex) { + ex.printStackTrace(); + throw new RuntimeException("main exception", ex); + } } private void setIdleLimit(String option) { @@ -192,6 +186,10 @@ private void setValidateMetadata(boolean validateMetadata) { this.validateMetadata = validateMetadata; } + private void setDeviceList(List deviceList) { + this.deviceList = deviceList; + } + private void setFeedTopic(String feedTopic) { System.err.println("Sending device feed to topic " + feedTopic); feedPusher = new PubSubPusher(projectId, feedTopic); @@ -279,7 +277,7 @@ private String getGenerationString() { } private void processDevices() { - processDevices(null); + processDevices(this.deviceList); } private void processDevices(List devices) { diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/RegistrarTest.java b/validator/src/main/java/com/google/daq/mqtt/registrar/RegistrarTest.java index d9b331de04..df014d3426 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/RegistrarTest.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/RegistrarTest.java @@ -72,7 +72,8 @@ private RegistrarUnderTest getRegistrarUnderTest() { ArrayList argList = new ArrayList(); argList.add("-s"); argList.add(SITE_PATH); - registrar.executeWithRegistrar(registrar, argList); + Registrar.processArgs(argList, registrar); + registrar.execute(); assertErrorSummaryValidateSuccess(registrar.getLastErrorSummary()); } @@ -83,7 +84,8 @@ private RegistrarUnderTest getRegistrarUnderTest() { argList.add("-t"); argList.add("-s"); argList.add(SITE_PATH); - registrar.executeWithRegistrar(registrar, argList); + Registrar.processArgs(argList, registrar); + registrar.execute(); assertErrorSummaryValidateFailure(registrar.getLastErrorSummary()); } From 7afa5b0fd5fceb90b50a376ca620f491ed6d59fe Mon Sep 17 00:00:00 2001 From: John Randolph Date: Thu, 23 Jun 2022 11:46:51 -0400 Subject: [PATCH 11/57] Import --- .../main/java/com/google/daq/mqtt/registrar/RegistrarTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/RegistrarTest.java b/validator/src/main/java/com/google/daq/mqtt/registrar/RegistrarTest.java index df014d3426..1bd6682dc8 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/RegistrarTest.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/RegistrarTest.java @@ -18,6 +18,7 @@ import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.TreeMap; import org.junit.Test; From fdfda2ed259d7374d1c084a7f1879134a65b1f53 Mon Sep 17 00:00:00 2001 From: John Randolph Date: Thu, 23 Jun 2022 14:44:37 -0400 Subject: [PATCH 12/57] Lint fixes --- .../com/google/daq/mqtt/registrar/LocalDevice.java | 3 ++- .../com/google/daq/mqtt/registrar/RegistrarTest.java | 10 +++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java b/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java index d2b6df87da..1404e01102 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java @@ -234,7 +234,8 @@ class LocalDevice { this(siteDir, devicesDir, deviceId, schemas, generation, null); } - public static void parseMetadataValidateProcessingReport(ProcessingReport report) throws ValidationException { + public static void parseMetadataValidateProcessingReport(ProcessingReport report) + throws ValidationException { if (report.isSuccess()) { return; } diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/RegistrarTest.java b/validator/src/main/java/com/google/daq/mqtt/registrar/RegistrarTest.java index 1bd6682dc8..9229ebb294 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/RegistrarTest.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/RegistrarTest.java @@ -43,8 +43,8 @@ protected JsonSchema getJsonSchema(String schemaName) { } private void assertErrorSummaryValidateSuccess(Map> summary) { - if ((summary == null) || (summary.get("Validating") == null) || - (summary.get("Validating").size() == 0)) { + if ((summary == null) || (summary.get("Validating") == null) + || (summary.get("Validating").size() == 0)) { return; } fail(summary.get("Validating").toString()); @@ -54,7 +54,7 @@ private void assertErrorSummaryValidateFailure(Map> if ((summary == null) || (summary.get("Validating") == null)) { fail("Error summary for Validating key is null"); } - if (summary.get("Validating").size()==0) { + if (summary.get("Validating").size() == 0) { fail("Error summary for Validating key is size 0"); } } @@ -68,7 +68,7 @@ private RegistrarUnderTest getRegistrarUnderTest() { } @Test public void metadataValidateSuccessTest() { - RegistrarUnderTest registrar = getRegistrarUnderTest(); + final RegistrarUnderTest registrar = getRegistrarUnderTest(); ArrayList argList = new ArrayList(); argList.add("-s"); @@ -79,7 +79,7 @@ private RegistrarUnderTest getRegistrarUnderTest() { } @Test public void metadataValidateFailureTest() { - RegistrarUnderTest registrar = getRegistrarUnderTest(); + final RegistrarUnderTest registrar = getRegistrarUnderTest(); ArrayList argList = new ArrayList(); argList.add("-t"); From 45ea044c6106fbf1bd6ac1198a1e3799205bb131 Mon Sep 17 00:00:00 2001 From: John Randolph Date: Thu, 23 Jun 2022 15:03:02 -0400 Subject: [PATCH 13/57] Up the warnings --- validator/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/build.gradle b/validator/build.gradle index 0d02cd3b0b..d485296cab 100644 --- a/validator/build.gradle +++ b/validator/build.gradle @@ -32,7 +32,7 @@ sourceSets { checkstyle { ignoreFailures = false - maxWarnings = 40 + maxWarnings = 50 } checkstyleMain.source = 'src/main/java' From 645b3fbbbd77e03d24e67eb9c3743060255820d4 Mon Sep 17 00:00:00 2001 From: Trevor Date: Mon, 27 Jun 2022 11:35:06 -0700 Subject: [PATCH 14/57] Basic Pubber Redirect (#376) --- .../java/daq/pubber/ConfigurationOptions.java | 1 + pubber/src/main/java/daq/pubber/Pubber.java | 28 +++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/pubber/src/main/java/daq/pubber/ConfigurationOptions.java b/pubber/src/main/java/daq/pubber/ConfigurationOptions.java index cc2557cdbe..111c6b958f 100644 --- a/pubber/src/main/java/daq/pubber/ConfigurationOptions.java +++ b/pubber/src/main/java/daq/pubber/ConfigurationOptions.java @@ -13,6 +13,7 @@ public class ConfigurationOptions { public Boolean noHardware; public String extraPoint; public String extraField; + public String redirectRegistry; /** * Returns a string of enabled options and values. diff --git a/pubber/src/main/java/daq/pubber/Pubber.java b/pubber/src/main/java/daq/pubber/Pubber.java index ad63bfbb3a..b9b4e9bd61 100644 --- a/pubber/src/main/java/daq/pubber/Pubber.java +++ b/pubber/src/main/java/daq/pubber/Pubber.java @@ -478,6 +478,8 @@ private void sendMessages() { updatePoints(); sendDeviceMessage(); flushDirtyState(); + // Some things can't be done from a on-message callback, so do them here instead. + maybeRedirectEndpoint(); } catch (Exception e) { error("Fatal error during execution", e); terminate(); @@ -547,6 +549,16 @@ private void initialize() { throw new RuntimeException("While creating out dir " + outDir.getPath(), e); } + initializeMqtt(); + } + + private void disconnectMqtt() { + Preconditions.checkState(mqttPublisher != null, "mqttPublisher not defined"); + mqttPublisher.close(); + mqttPublisher = null; + } + + private void initializeMqtt() { Preconditions.checkNotNull(configuration.deviceId, "configuration deviceId not defined"); if (configuration.sitePath != null && configuration.keyFile != null) { String keyDevice = @@ -692,6 +704,22 @@ private void processConfigUpdate(Config config) { maybeRestartExecutor(actualInterval); } + private void maybeRedirectEndpoint() { + String redirectRegistry = configuration.options.redirectRegistry; + if (redirectRegistry == null || redirectRegistry.equals(configuration.registryId) + || configLatch.getCount() > 0) { + return; + } + try { + disconnectMqtt(); + configuration.registryId = redirectRegistry; + initializeMqtt(); + startConnection(onDone); + } catch (Exception e) { + throw new RuntimeException("While redirecting connection endpoint", e); + } + } + private void updateDiscoveryConfig(DiscoveryConfig discovery) { DiscoveryConfig discoveryConfig = discovery == null ? new DiscoveryConfig() : discovery; if (deviceState.discovery == null) { From ffd812efd5b429e2f50f914b70cfe642cc764e44 Mon Sep 17 00:00:00 2001 From: John Randolph Date: Tue, 28 Jun 2022 12:45:29 -0400 Subject: [PATCH 15/57] Add preliminary base64 payload prep/apply/commit blob msgs. --- tests/config.tests/apply.json | 12 ++++++------ tests/config.tests/commit.json | 12 ++++++------ tests/config.tests/prep.json | 14 ++++++++++++++ 3 files changed, 26 insertions(+), 12 deletions(-) create mode 100644 tests/config.tests/prep.json diff --git a/tests/config.tests/apply.json b/tests/config.tests/apply.json index 38b8f7f55b..8c985ac5bf 100644 --- a/tests/config.tests/apply.json +++ b/tests/config.tests/apply.json @@ -1,12 +1,12 @@ { "version": 1, "blobset": { - "blobs": { - "codebase": { - "stage": "apply", - "target": "e21637e422d3153b866279af92ec1b1fd17c8846" + "phase": "apply", + "blobs": [ + { + "id": "hello_playload" } - } + ] }, - "timestamp": "2021-06-17T12:20:44.254Z" + "timestamp": "2022-06-28T12:00:00.000Z" } diff --git a/tests/config.tests/commit.json b/tests/config.tests/commit.json index c2433d8f75..a8c38fa2fc 100644 --- a/tests/config.tests/commit.json +++ b/tests/config.tests/commit.json @@ -1,12 +1,12 @@ { "version": 1, "blobset": { - "blobs": { - "codebase": { - "stage": "steady", - "target": "1623941564.8938339" + "phase": "commit", + "blobs": [ + { + "id": "hello_playload" } - } + ] }, - "timestamp": "2021-06-17T12:20:44.254Z" + "timestamp": "2022-06-28T12:00:00.000Z" } diff --git a/tests/config.tests/prep.json b/tests/config.tests/prep.json new file mode 100644 index 0000000000..3e8547fca0 --- /dev/null +++ b/tests/config.tests/prep.json @@ -0,0 +1,14 @@ +{ + "version": 1, + "blobset": { + "phase": "prep", + "blobs": [ + { + "id": "hello_payload", + "type": "base64", + "payload": "SEVMTE8=" + } + ] + }, + "timestamp": "2022-06-28T12:00:00.000Z" +} From a1ae005f903d22d78aad382cab2aad586e900210 Mon Sep 17 00:00:00 2001 From: John Randolph Date: Tue, 28 Jun 2022 13:18:12 -0400 Subject: [PATCH 16/57] Proposal that it may be beneficial to hint the type that base64 decode will result in. It could be ascii, utf8, binary, etc. --- tests/config.tests/prep.json | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/config.tests/prep.json b/tests/config.tests/prep.json index 3e8547fca0..75c761f71f 100644 --- a/tests/config.tests/prep.json +++ b/tests/config.tests/prep.json @@ -6,6 +6,7 @@ { "id": "hello_payload", "type": "base64", + "content_type": "text/plain", "payload": "SEVMTE8=" } ] From 65771ad5552aacd2fb94f1200994621dec3286ca Mon Sep 17 00:00:00 2001 From: John Randolph Date: Tue, 28 Jun 2022 15:15:18 -0400 Subject: [PATCH 17/57] Model ids as map key name --- tests/config.tests/fetch.json | 12 ------------ tests/config.tests/prep.json | 3 +-- 2 files changed, 1 insertion(+), 14 deletions(-) delete mode 100644 tests/config.tests/fetch.json diff --git a/tests/config.tests/fetch.json b/tests/config.tests/fetch.json deleted file mode 100644 index fcc98a6899..0000000000 --- a/tests/config.tests/fetch.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "version": 1, - "blobset": { - "blobs": { - "codebase": { - "stage": "fetch", - "target": "1.4" - } - } - }, - "timestamp": "2021-06-17T12:20:44.254Z" -} diff --git a/tests/config.tests/prep.json b/tests/config.tests/prep.json index 75c761f71f..666f0079f3 100644 --- a/tests/config.tests/prep.json +++ b/tests/config.tests/prep.json @@ -3,8 +3,7 @@ "blobset": { "phase": "prep", "blobs": [ - { - "id": "hello_payload", + "hello_payload": "type": "base64", "content_type": "text/plain", "payload": "SEVMTE8=" From cf7731bcce00c430689baf294a1ca318d4b1f70c Mon Sep 17 00:00:00 2001 From: John Randolph Date: Tue, 28 Jun 2022 15:16:06 -0400 Subject: [PATCH 18/57] Model map key name as id --- tests/config.tests/apply.json | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/config.tests/apply.json b/tests/config.tests/apply.json index 8c985ac5bf..e71de27035 100644 --- a/tests/config.tests/apply.json +++ b/tests/config.tests/apply.json @@ -2,11 +2,9 @@ "version": 1, "blobset": { "phase": "apply", - "blobs": [ - { - "id": "hello_playload" - } - ] + "blobs": { + "hello_payload": {} + } }, "timestamp": "2022-06-28T12:00:00.000Z" } From 0f38b05ca9a03b2d0eebf8258d849088ba041db7 Mon Sep 17 00:00:00 2001 From: John Randolph Date: Tue, 28 Jun 2022 15:17:27 -0400 Subject: [PATCH 19/57] Prep --- tests/config.tests/prep.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/config.tests/prep.json b/tests/config.tests/prep.json index 666f0079f3..d3cb8852c2 100644 --- a/tests/config.tests/prep.json +++ b/tests/config.tests/prep.json @@ -2,13 +2,13 @@ "version": 1, "blobset": { "phase": "prep", - "blobs": [ - "hello_payload": + "blobs": { + "hello_payload": { "type": "base64", "content_type": "text/plain", "payload": "SEVMTE8=" } - ] + } }, "timestamp": "2022-06-28T12:00:00.000Z" } From 6144588671157166bac8e7dde02a8f73b1e03f83 Mon Sep 17 00:00:00 2001 From: Trevor Date: Tue, 28 Jun 2022 17:23:51 -0400 Subject: [PATCH 20/57] Refactor pubber endpoint config (#377) --- bin/pubber | 6 ++++-- .../main/java/daq/pubber/Configuration.java | 6 +----- .../daq/pubber/EndpointConfiguration.java | 13 +++++++++++++ .../main/java/daq/pubber/MqttPublisher.java | 19 +++++++++++-------- pubber/src/main/java/daq/pubber/Pubber.java | 10 +++++----- 5 files changed, 34 insertions(+), 20 deletions(-) create mode 100644 pubber/src/main/java/daq/pubber/EndpointConfiguration.java diff --git a/bin/pubber b/bin/pubber index 39eaf74437..55cc8173ba 100755 --- a/bin/pubber +++ b/bin/pubber @@ -43,9 +43,11 @@ else cat < /tmp/pubber_config.json { - "projectId": "$project_id", + "endpoint": { + "projectId": "$project_id", + "cloudRegion": "$cloud_region" + }, "sitePath": "$site_path", - "cloudRegion": "$cloud_region", "deviceId": "$device_id", "serialNo": "$serial_no", "options": $options_json diff --git a/pubber/src/main/java/daq/pubber/Configuration.java b/pubber/src/main/java/daq/pubber/Configuration.java index 2232800947..feae4bd908 100644 --- a/pubber/src/main/java/daq/pubber/Configuration.java +++ b/pubber/src/main/java/daq/pubber/Configuration.java @@ -5,11 +5,7 @@ * General bucket of pubber configuration information. */ public class Configuration { - public String bridgeHostname = "mqtt.googleapis.com"; - public String bridgePort = "443"; - public String projectId; - public String cloudRegion; - public String registryId; + public EndpointConfiguration endpoint = new EndpointConfiguration(); public String gatewayId; public String deviceId; public String sitePath; diff --git a/pubber/src/main/java/daq/pubber/EndpointConfiguration.java b/pubber/src/main/java/daq/pubber/EndpointConfiguration.java new file mode 100644 index 0000000000..947e5b0d5d --- /dev/null +++ b/pubber/src/main/java/daq/pubber/EndpointConfiguration.java @@ -0,0 +1,13 @@ +package daq.pubber; + +/** + * Configuration for the connection endpoint. + */ +public class EndpointConfiguration { + + public String bridgeHostname = "mqtt.googleapis.com"; + public String bridgePort = "443"; + public String cloudRegion; + public String projectId; + public String registryId; +} diff --git a/pubber/src/main/java/daq/pubber/MqttPublisher.java b/pubber/src/main/java/daq/pubber/MqttPublisher.java index 93666efbeb..fdcbd12daf 100644 --- a/pubber/src/main/java/daq/pubber/MqttPublisher.java +++ b/pubber/src/main/java/daq/pubber/MqttPublisher.java @@ -92,7 +92,7 @@ public class MqttPublisher { MqttPublisher(Configuration configuration, Consumer onError) { this.configuration = configuration; - this.registryId = configuration.registryId; + this.registryId = configuration.endpoint.registryId; this.onError = onError; validateCloudIotOptions(); } @@ -157,10 +157,10 @@ void close() { private void validateCloudIotOptions() { try { - checkNotNull(configuration.bridgeHostname, "bridgeHostname"); - checkNotNull(configuration.bridgePort, "bridgePort"); - checkNotNull(configuration.projectId, "projectId"); - checkNotNull(configuration.cloudRegion, "cloudRegion"); + checkNotNull(configuration.endpoint.bridgeHostname, "bridgeHostname"); + checkNotNull(configuration.endpoint.bridgePort, "bridgePort"); + checkNotNull(configuration.endpoint.projectId, "projectId"); + checkNotNull(configuration.endpoint.cloudRegion, "cloudRegion"); checkNotNull(configuration.keyBytes, "keyBytes"); checkNotNull(configuration.algorithm, "algorithm"); } catch (Exception e) { @@ -237,7 +237,8 @@ private MqttClient connectMqttClient(String deviceId) { } private char[] createJwt() throws Exception { - return createJwt(configuration.projectId, configuration.keyBytes, configuration.algorithm) + return createJwt(configuration.endpoint.projectId, configuration.keyBytes, + configuration.algorithm) .toCharArray(); } @@ -271,14 +272,16 @@ private String createJwt(String projectId, byte[] privateKeyBytes, String algori private String getClientId(String deviceId) { // Create our MQTT client. The mqttClientId is a unique string that identifies this device. For // Google Cloud IoT, it must be in the format below. - return String.format(ID_FORMAT, configuration.projectId, configuration.cloudRegion, + return String.format(ID_FORMAT, configuration.endpoint.projectId, + configuration.endpoint.cloudRegion, registryId, deviceId); } private String getBrokerUrl() { // Build the connection string for Google's Cloud IoT MQTT server. Only SSL connections are // accepted. For server authentication, the JVM's root certificates are used. - return String.format(BROKER_URL_FORMAT, configuration.bridgeHostname, configuration.bridgePort); + return String.format(BROKER_URL_FORMAT, configuration.endpoint.bridgeHostname, + configuration.endpoint.bridgePort); } private String getMessageTopic(String deviceId, String topic) { diff --git a/pubber/src/main/java/daq/pubber/Pubber.java b/pubber/src/main/java/daq/pubber/Pubber.java index b9b4e9bd61..25d4e618ce 100644 --- a/pubber/src/main/java/daq/pubber/Pubber.java +++ b/pubber/src/main/java/daq/pubber/Pubber.java @@ -164,7 +164,7 @@ public Pubber(String configPath) { */ public Pubber(String projectId, String sitePath, String deviceId, String serialNo) { configuration = new Configuration(); - configuration.projectId = projectId; + configuration.endpoint.projectId = projectId; configuration.deviceId = deviceId; configuration.serialNo = serialNo; if (PUBSUB_SITE.equals(sitePath)) { @@ -342,8 +342,8 @@ private void loadCloudConfig() { } private void processCloudConfig(CloudIotConfig cloudIotConfig) { - configuration.registryId = cloudIotConfig.registry_id; - configuration.cloudRegion = cloudIotConfig.cloud_region; + configuration.endpoint.registryId = cloudIotConfig.registry_id; + configuration.endpoint.cloudRegion = cloudIotConfig.cloud_region; } private void initializeDevice() { @@ -706,13 +706,13 @@ private void processConfigUpdate(Config config) { private void maybeRedirectEndpoint() { String redirectRegistry = configuration.options.redirectRegistry; - if (redirectRegistry == null || redirectRegistry.equals(configuration.registryId) + if (redirectRegistry == null || redirectRegistry.equals(configuration.endpoint.registryId) || configLatch.getCount() > 0) { return; } try { disconnectMqtt(); - configuration.registryId = redirectRegistry; + configuration.endpoint.registryId = redirectRegistry; initializeMqtt(); startConnection(onDone); } catch (Exception e) { From 520733f965dd0207f5cd9b051bc65fd26583977e Mon Sep 17 00:00:00 2001 From: John Randolph Date: Wed, 29 Jun 2022 10:29:50 -0400 Subject: [PATCH 21/57] Updates for blobset = map of blobs{id}, phase inside blob --- tests/config.tests/apply.json | 5 +++-- tests/config.tests/commit.json | 9 ++++----- tests/config.tests/prep.json | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/config.tests/apply.json b/tests/config.tests/apply.json index e71de27035..5cecbc9ae8 100644 --- a/tests/config.tests/apply.json +++ b/tests/config.tests/apply.json @@ -1,9 +1,10 @@ { "version": 1, "blobset": { - "phase": "apply", "blobs": { - "hello_payload": {} + "hello_payload": { + "phase": "apply" + } } }, "timestamp": "2022-06-28T12:00:00.000Z" diff --git a/tests/config.tests/commit.json b/tests/config.tests/commit.json index a8c38fa2fc..96d27d4faf 100644 --- a/tests/config.tests/commit.json +++ b/tests/config.tests/commit.json @@ -1,12 +1,11 @@ { "version": 1, "blobset": { - "phase": "commit", - "blobs": [ - { - "id": "hello_playload" + "blobs": { + "hello_payload": { + "phase": "commit" } - ] + } }, "timestamp": "2022-06-28T12:00:00.000Z" } diff --git a/tests/config.tests/prep.json b/tests/config.tests/prep.json index d3cb8852c2..900f30d52f 100644 --- a/tests/config.tests/prep.json +++ b/tests/config.tests/prep.json @@ -1,9 +1,9 @@ { "version": 1, "blobset": { - "phase": "prep", "blobs": { "hello_payload": { + "phase": "prep", "type": "base64", "content_type": "text/plain", "payload": "SEVMTE8=" From bf73b634fb2941346fc1ce2b49b9ee1491c954e4 Mon Sep 17 00:00:00 2001 From: John Randolph Date: Wed, 29 Jun 2022 10:35:46 -0400 Subject: [PATCH 22/57] Add type=url for retrieving http/https resources --- tests/config.tests/prep.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/config.tests/prep.json b/tests/config.tests/prep.json index 900f30d52f..7a6270fc02 100644 --- a/tests/config.tests/prep.json +++ b/tests/config.tests/prep.json @@ -7,6 +7,11 @@ "type": "base64", "content_type": "text/plain", "payload": "SEVMTE8=" + }, + "update_payload": { + "phase": "prep", + "type": "url", + "href": "http://manuf/path/file" } } }, From dca05b52bd89603226c73db1e347715d7351ebcd Mon Sep 17 00:00:00 2001 From: John Randolph Date: Wed, 29 Jun 2022 11:10:43 -0400 Subject: [PATCH 23/57] Add two examples of retrieving a zip --- tests/config.tests/prep.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/config.tests/prep.json b/tests/config.tests/prep.json index 7a6270fc02..136478bac5 100644 --- a/tests/config.tests/prep.json +++ b/tests/config.tests/prep.json @@ -2,16 +2,16 @@ "version": 1, "blobset": { "blobs": { - "hello_payload": { + "fw_payload": { "phase": "prep", "type": "base64", - "content_type": "text/plain", - "payload": "SEVMTE8=" + "content_type": "application/zip", + "payload": "UEsDBAoAAAAAAAZZ3VSU7ZiuBQAAAAUAAAAIABwAYm9vdC5pbWdVVAkAA9xqvGLcarxidXgLAAEEFjQAAARTXwEAYm9vdApQSwMECgAAAAAAB1ndVK9osDMDAAAAAwAAAAYAHABmdy5pbWdVVAkAA95qvGLearxidXgLAAEEFjQAAARTXwEAZncKUEsBAh4DCgAAAAAABlndVJTtmK4FAAAABQAAAAgAGAAAAAAAAQAAAKSBAAAAAGJvb3QuaW1nVVQFAAPcarxidXgLAAEEFjQAAARTXwEAUEsBAh4DCgAAAAAAB1ndVK9osDMDAAAAAwAAAAYAGAAAAAAAAQAAAKSBRwAAAGZ3LmltZ1VUBQAD3mq8YnV4CwABBBY0AAAEU18BAFBLBQYAAAAAAgACAJoAAACKAAAAAAA=" }, "update_payload": { "phase": "prep", "type": "url", - "href": "http://manuf/path/file" + "href": "http://manuf/path/file.zip" } } }, From 304b8294a5ef423619b809d2bcd886020151ff55 Mon Sep 17 00:00:00 2001 From: John Randolph Date: Wed, 29 Jun 2022 18:13:28 -0400 Subject: [PATCH 24/57] Add proposed format for a config blob in json --- tests/config.tests/prep.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/config.tests/prep.json b/tests/config.tests/prep.json index 136478bac5..ee0b41a37e 100644 --- a/tests/config.tests/prep.json +++ b/tests/config.tests/prep.json @@ -12,6 +12,12 @@ "phase": "prep", "type": "url", "href": "http://manuf/path/file.zip" + }, + "endpoint_update": { + "phase": "prep", + "type": "base64", + "content_type": "application/json", + "payload": "e30=" } } }, From 253dcaa893281cde139f17e3f8c31b2bff979d27 Mon Sep 17 00:00:00 2001 From: John Randolph Date: Thu, 30 Jun 2022 16:44:57 -0400 Subject: [PATCH 25/57] Change to base64: "inline_payload", url: "url", etc. --- tests/config.tests/prep.json | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/config.tests/prep.json b/tests/config.tests/prep.json index ee0b41a37e..d05dad92bb 100644 --- a/tests/config.tests/prep.json +++ b/tests/config.tests/prep.json @@ -4,20 +4,17 @@ "blobs": { "fw_payload": { "phase": "prep", - "type": "base64", "content_type": "application/zip", - "payload": "UEsDBAoAAAAAAAZZ3VSU7ZiuBQAAAAUAAAAIABwAYm9vdC5pbWdVVAkAA9xqvGLcarxidXgLAAEEFjQAAARTXwEAYm9vdApQSwMECgAAAAAAB1ndVK9osDMDAAAAAwAAAAYAHABmdy5pbWdVVAkAA95qvGLearxidXgLAAEEFjQAAARTXwEAZncKUEsBAh4DCgAAAAAABlndVJTtmK4FAAAABQAAAAgAGAAAAAAAAQAAAKSBAAAAAGJvb3QuaW1nVVQFAAPcarxidXgLAAEEFjQAAARTXwEAUEsBAh4DCgAAAAAAB1ndVK9osDMDAAAAAwAAAAYAGAAAAAAAAQAAAKSBRwAAAGZ3LmltZ1VUBQAD3mq8YnV4CwABBBY0AAAEU18BAFBLBQYAAAAAAgACAJoAAACKAAAAAAA=" + "base64": "UEsDBAoAAAAAAAZZ3VSU7ZiuBQAAAAUAAAAIABwAYm9vdC5pbWdVVAkAA9xqvGLcarxidXgLAAEEFjQAAARTXwEAYm9vdApQSwMECgAAAAAAB1ndVK9osDMDAAAAAwAAAAYAHABmdy5pbWdVVAkAA95qvGLearxidXgLAAEEFjQAAARTXwEAZncKUEsBAh4DCgAAAAAABlndVJTtmK4FAAAABQAAAAgAGAAAAAAAAQAAAKSBAAAAAGJvb3QuaW1nVVQFAAPcarxidXgLAAEEFjQAAARTXwEAUEsBAh4DCgAAAAAAB1ndVK9osDMDAAAAAwAAAAYAGAAAAAAAAQAAAKSBRwAAAGZ3LmltZ1VUBQAD3mq8YnV4CwABBBY0AAAEU18BAFBLBQYAAAAAAgACAJoAAACKAAAAAAA=" }, "update_payload": { "phase": "prep", - "type": "url", - "href": "http://manuf/path/file.zip" + "url": "http://manuf/path/file.zip" }, "endpoint_update": { "phase": "prep", - "type": "base64", "content_type": "application/json", - "payload": "e30=" + "base64": "e30=" } } }, From b95edfcff3941c04af6badbf6e9b79d0f0e9e3b2 Mon Sep 17 00:00:00 2001 From: John Randolph Date: Fri, 1 Jul 2022 11:18:52 -0400 Subject: [PATCH 26/57] Update examples for apply and commit --- tests/config.tests/apply.json | 2 +- tests/config.tests/commit.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/config.tests/apply.json b/tests/config.tests/apply.json index 5cecbc9ae8..ffed8ab3a3 100644 --- a/tests/config.tests/apply.json +++ b/tests/config.tests/apply.json @@ -2,7 +2,7 @@ "version": 1, "blobset": { "blobs": { - "hello_payload": { + "endpoint_update": { "phase": "apply" } } diff --git a/tests/config.tests/commit.json b/tests/config.tests/commit.json index 96d27d4faf..22c35079bf 100644 --- a/tests/config.tests/commit.json +++ b/tests/config.tests/commit.json @@ -2,7 +2,7 @@ "version": 1, "blobset": { "blobs": { - "hello_payload": { + "endpoint_update": { "phase": "commit" } } From 32ab6b57af498890ac1430c337a8ab1cba368d7d Mon Sep 17 00:00:00 2001 From: John R Date: Wed, 13 Jul 2022 12:56:27 -0400 Subject: [PATCH 27/57] Update commit.json --- tests/config.tests/commit.json | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/tests/config.tests/commit.json b/tests/config.tests/commit.json index 22c35079bf..12c0a7bc7c 100644 --- a/tests/config.tests/commit.json +++ b/tests/config.tests/commit.json @@ -2,10 +2,22 @@ "version": 1, "blobset": { "blobs": { - "endpoint_update": { - "phase": "commit" + "arbitrary_manufacturer_id": { + "phase": "commit", + "content_type": "application/zip", + "base64": "UEsDBAoAAAAAAAZZ3VSU7ZiuBQAAAAUAAAAIABwAYm9vdC5pbWdVVAkAA9xqvGLcarxidXgLAAEEFjQAAARTXwEAYm9vdApQSwMECgAAAAAAB1ndVK9osDMDAAAAAwAAAAYAHABmdy5pbWdVVAkAA95qvGLearxidXgLAAEEFjQAAARTXwEAZncKUEsBAh4DCgAAAAAABlndVJTtmK4FAAAABQAAAAgAGAAAAAAAAQAAAKSBAAAAAGJvb3QuaW1nVVQFAAPcarxidXgLAAEEFjQAAARTXwEAUEsBAh4DCgAAAAAAB1ndVK9osDMDAAAAAwAAAAYAGAAAAAAAAQAAAKSBRwAAAGZ3LmltZ1VUBQAD3mq8YnV4CwABBBY0AAAEU18BAFBLBQYAAAAAAgACAJoAAACKAAAAAAA=" + }, + "_firmware_update": { + "phase": "commit", + "url": "https://manuf.com/path/file.zip", + "hash": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "_endpoint_redirect": { + "phase": "commit", + "content_type": "application/json", + "base64": "e30=" } } }, - "timestamp": "2022-06-28T12:00:00.000Z" + "timestamp": "2022-07-13T12:00:00.000Z" } From 421cb9765b8934d8a483963439c44d7b75a5d622 Mon Sep 17 00:00:00 2001 From: John R Date: Wed, 13 Jul 2022 12:56:52 -0400 Subject: [PATCH 28/57] Delete prep.json --- tests/config.tests/prep.json | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 tests/config.tests/prep.json diff --git a/tests/config.tests/prep.json b/tests/config.tests/prep.json deleted file mode 100644 index d05dad92bb..0000000000 --- a/tests/config.tests/prep.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "version": 1, - "blobset": { - "blobs": { - "fw_payload": { - "phase": "prep", - "content_type": "application/zip", - "base64": "UEsDBAoAAAAAAAZZ3VSU7ZiuBQAAAAUAAAAIABwAYm9vdC5pbWdVVAkAA9xqvGLcarxidXgLAAEEFjQAAARTXwEAYm9vdApQSwMECgAAAAAAB1ndVK9osDMDAAAAAwAAAAYAHABmdy5pbWdVVAkAA95qvGLearxidXgLAAEEFjQAAARTXwEAZncKUEsBAh4DCgAAAAAABlndVJTtmK4FAAAABQAAAAgAGAAAAAAAAQAAAKSBAAAAAGJvb3QuaW1nVVQFAAPcarxidXgLAAEEFjQAAARTXwEAUEsBAh4DCgAAAAAAB1ndVK9osDMDAAAAAwAAAAYAGAAAAAAAAQAAAKSBRwAAAGZ3LmltZ1VUBQAD3mq8YnV4CwABBBY0AAAEU18BAFBLBQYAAAAAAgACAJoAAACKAAAAAAA=" - }, - "update_payload": { - "phase": "prep", - "url": "http://manuf/path/file.zip" - }, - "endpoint_update": { - "phase": "prep", - "content_type": "application/json", - "base64": "e30=" - } - } - }, - "timestamp": "2022-06-28T12:00:00.000Z" -} From 8433c336114cc2d7f4da304d6a93608a0b8fe58d Mon Sep 17 00:00:00 2001 From: John R Date: Wed, 13 Jul 2022 12:57:23 -0400 Subject: [PATCH 29/57] Delete apply.json --- tests/config.tests/apply.json | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 tests/config.tests/apply.json diff --git a/tests/config.tests/apply.json b/tests/config.tests/apply.json deleted file mode 100644 index ffed8ab3a3..0000000000 --- a/tests/config.tests/apply.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "version": 1, - "blobset": { - "blobs": { - "endpoint_update": { - "phase": "apply" - } - } - }, - "timestamp": "2022-06-28T12:00:00.000Z" -} From c3482597fb0c02103735e77c4f3d82320e4410f7 Mon Sep 17 00:00:00 2001 From: John Randolph Date: Thu, 14 Jul 2022 10:00:32 -0400 Subject: [PATCH 30/57] Change hash to sha256 --- tests/config.tests/commit.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/config.tests/commit.json b/tests/config.tests/commit.json index 12c0a7bc7c..e49b705153 100644 --- a/tests/config.tests/commit.json +++ b/tests/config.tests/commit.json @@ -10,7 +10,7 @@ "_firmware_update": { "phase": "commit", "url": "https://manuf.com/path/file.zip", - "hash": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, "_endpoint_redirect": { "phase": "commit", From 5089cbcf502a575ced1f56cbcc6cd26310eed0db Mon Sep 17 00:00:00 2001 From: John Randolph Date: Thu, 14 Jul 2022 12:21:16 -0400 Subject: [PATCH 31/57] Rename commit to applied --- tests/config.tests/{commit.json => applied.json} | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename tests/config.tests/{commit.json => applied.json} (91%) diff --git a/tests/config.tests/commit.json b/tests/config.tests/applied.json similarity index 91% rename from tests/config.tests/commit.json rename to tests/config.tests/applied.json index e49b705153..8850cfe5f3 100644 --- a/tests/config.tests/commit.json +++ b/tests/config.tests/applied.json @@ -3,17 +3,17 @@ "blobset": { "blobs": { "arbitrary_manufacturer_id": { - "phase": "commit", + "phase": "applied", "content_type": "application/zip", "base64": "UEsDBAoAAAAAAAZZ3VSU7ZiuBQAAAAUAAAAIABwAYm9vdC5pbWdVVAkAA9xqvGLcarxidXgLAAEEFjQAAARTXwEAYm9vdApQSwMECgAAAAAAB1ndVK9osDMDAAAAAwAAAAYAHABmdy5pbWdVVAkAA95qvGLearxidXgLAAEEFjQAAARTXwEAZncKUEsBAh4DCgAAAAAABlndVJTtmK4FAAAABQAAAAgAGAAAAAAAAQAAAKSBAAAAAGJvb3QuaW1nVVQFAAPcarxidXgLAAEEFjQAAARTXwEAUEsBAh4DCgAAAAAAB1ndVK9osDMDAAAAAwAAAAYAGAAAAAAAAQAAAKSBRwAAAGZ3LmltZ1VUBQAD3mq8YnV4CwABBBY0AAAEU18BAFBLBQYAAAAAAgACAJoAAACKAAAAAAA=" }, "_firmware_update": { - "phase": "commit", + "phase": "applied", "url": "https://manuf.com/path/file.zip", "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, "_endpoint_redirect": { - "phase": "commit", + "phase": "applied", "content_type": "application/json", "base64": "e30=" } From 43e5ce01eb1af29bad201d8c2a4e46712130316b Mon Sep 17 00:00:00 2001 From: John Randolph Date: Thu, 14 Jul 2022 14:59:27 -0400 Subject: [PATCH 32/57] Beginning of writing state replies for blobset config msgs. --- tests/state.tests/blobset_base64_error.json | 12 ++++++++++++ tests/state.tests/blobset_received.json | 17 +++++++++++++++++ tests/state.tests/blobset_url_error.json | 13 +++++++++++++ 3 files changed, 42 insertions(+) create mode 100644 tests/state.tests/blobset_base64_error.json create mode 100644 tests/state.tests/blobset_received.json create mode 100644 tests/state.tests/blobset_url_error.json diff --git a/tests/state.tests/blobset_base64_error.json b/tests/state.tests/blobset_base64_error.json new file mode 100644 index 0000000000..dce5d8be4d --- /dev/null +++ b/tests/state.tests/blobset_base64_error.json @@ -0,0 +1,12 @@ +{ + "version": 1, + "timestamp": "2022-07-13T12:00:02.000Z", + "blobset": { + "blobs": { + "_endpoint_redirect": { + "phase": "error", + "base64": "base64 encoded error message from base64 decoder" + } + } + } +} diff --git a/tests/state.tests/blobset_received.json b/tests/state.tests/blobset_received.json new file mode 100644 index 0000000000..5417b355df --- /dev/null +++ b/tests/state.tests/blobset_received.json @@ -0,0 +1,17 @@ +{ + "version": 1, + "timestamp": "2022-07-13T12:00:02.000Z", + "blobset": { + "blobs": { + "arbitrary_manufacturer_id": { + "phase": "received" + }, + "_firmware_update": { + "phase": "received" + }, + "_endpoint_redirect": { + "phase": "received" + } + } + } +} diff --git a/tests/state.tests/blobset_url_error.json b/tests/state.tests/blobset_url_error.json new file mode 100644 index 0000000000..c8424e5b73 --- /dev/null +++ b/tests/state.tests/blobset_url_error.json @@ -0,0 +1,13 @@ +{ + "version": 1, + "timestamp": "2022-07-13T12:00:02.000Z", + "blobset": { + "blobs": { + "_firmware_update": { + "phase": "error", + "url": "https://manuf.com/path/file.zip", + "base64" "base64 encoded HTTPS error response message" + } + } + } +} From de98d669f166bca128abd3e18589a8ac8275f7ec Mon Sep 17 00:00:00 2001 From: John Randolph Date: Mon, 18 Jul 2022 16:31:09 -0400 Subject: [PATCH 33/57] Add further examples --- tests/state.tests/blobset_received.json | 28 +++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/tests/state.tests/blobset_received.json b/tests/state.tests/blobset_received.json index 5417b355df..3fb1e53a96 100644 --- a/tests/state.tests/blobset_received.json +++ b/tests/state.tests/blobset_received.json @@ -1,16 +1,36 @@ { "version": 1, - "timestamp": "2022-07-13T12:00:02.000Z", + "timestamp": "2022-07-13T12:00:10.000Z", + "system": {}, "blobset": { + "state_etag": "0", "blobs": { "arbitrary_manufacturer_id": { - "phase": "received" + "phase": "received", + "status": { + "message": "Received", + "category": "", + "timestamp": "2022-07-13T12:00:08.000Z", + "level": 100 + } }, "_firmware_update": { - "phase": "received" + "phase": "received", + "status": { + "message": "Received", + "category": "", + "timestamp": "2022-07-13T12:00:08.000Z", + "level": 100 + } }, "_endpoint_redirect": { - "phase": "received" + "phase": "received", + "status": { + "message": "Received", + "category": "", + "timestamp": "2022-07-13T12:00:08.000Z", + "level": 100 + } } } } From f603cb339b44d62be442911afb514a3e9a7a72dd Mon Sep 17 00:00:00 2001 From: John Randolph Date: Tue, 19 Jul 2022 14:45:12 -0400 Subject: [PATCH 34/57] Add category, message, remove phase --- schema/state_blobset.json | 4 ---- tests/state.tests/blobset_receive.json | 33 ++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 4 deletions(-) create mode 100644 tests/state.tests/blobset_receive.json diff --git a/schema/state_blobset.json b/schema/state_blobset.json index 004747e689..e74e503032 100644 --- a/schema/state_blobset.json +++ b/schema/state_blobset.json @@ -4,10 +4,6 @@ "$schema": "http://json-schema.org/draft-07/schema#", "additionalProperties": false, "properties": { - "state_etag": { - "type": "string", - "maxLength": 32 - }, "blobs": { "additionalProperties": false, "existingJavaType": "java.util.HashMap", diff --git a/tests/state.tests/blobset_receive.json b/tests/state.tests/blobset_receive.json new file mode 100644 index 0000000000..038630a9ee --- /dev/null +++ b/tests/state.tests/blobset_receive.json @@ -0,0 +1,33 @@ +{ + "version": 1, + "timestamp": "2022-07-13T12:00:10.000Z", + "system": {}, + "blobset": { + "blobs": { + "arbitrary_manufacturer_id": { + "status": { + "message": "Received arbitrary_manufacturer_id", + "category": "blobset.blobs.receive", + "timestamp": "2022-07-13T12:00:08.000Z", + "level": 100 + } + }, + "_firmware_update": { + "status": { + "message": "Received firmware update", + "category": "blobset.blobs.receive", + "timestamp": "2022-07-13T12:00:08.000Z", + "level": 100 + } + }, + "_endpoint_redirect": { + "status": { + "message": "Receivedi endpoint redirect", + "category": "blobset.blobs.receive", + "timestamp": "2022-07-13T12:00:08.000Z", + "level": 100 + } + } + } + } +} From cff3e9d847d642e2f0b18628e6c51cdef9792b30 Mon Sep 17 00:00:00 2001 From: John Randolph Date: Tue, 19 Jul 2022 15:06:32 -0400 Subject: [PATCH 35/57] rm --- tests/state.tests/blobset_received.json | 37 ------------------------- 1 file changed, 37 deletions(-) delete mode 100644 tests/state.tests/blobset_received.json diff --git a/tests/state.tests/blobset_received.json b/tests/state.tests/blobset_received.json deleted file mode 100644 index 3fb1e53a96..0000000000 --- a/tests/state.tests/blobset_received.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "version": 1, - "timestamp": "2022-07-13T12:00:10.000Z", - "system": {}, - "blobset": { - "state_etag": "0", - "blobs": { - "arbitrary_manufacturer_id": { - "phase": "received", - "status": { - "message": "Received", - "category": "", - "timestamp": "2022-07-13T12:00:08.000Z", - "level": 100 - } - }, - "_firmware_update": { - "phase": "received", - "status": { - "message": "Received", - "category": "", - "timestamp": "2022-07-13T12:00:08.000Z", - "level": 100 - } - }, - "_endpoint_redirect": { - "phase": "received", - "status": { - "message": "Received", - "category": "", - "timestamp": "2022-07-13T12:00:08.000Z", - "level": 100 - } - } - } - } -} From 42dc8ba558d5eb49013febf92fee4b5c2adc1d0a Mon Sep 17 00:00:00 2001 From: John Randolph Date: Tue, 19 Jul 2022 15:11:05 -0400 Subject: [PATCH 36/57] Add required fields --- tests/state.tests/blobset_receive.json | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/tests/state.tests/blobset_receive.json b/tests/state.tests/blobset_receive.json index 038630a9ee..5d821208dd 100644 --- a/tests/state.tests/blobset_receive.json +++ b/tests/state.tests/blobset_receive.json @@ -1,7 +1,18 @@ { "version": 1, "timestamp": "2022-07-13T12:00:10.000Z", - "system": {}, + "system": { + "hardware": { + "make": "ACME", + "model": "Bird Trap" + }, + "software": { + "firmware": "1.0" + }, + "serial_no": "000000", + "last_config": "2022-07-11T00:00:10.000Z", + "operational": true + }, "blobset": { "blobs": { "arbitrary_manufacturer_id": { @@ -22,7 +33,7 @@ }, "_endpoint_redirect": { "status": { - "message": "Receivedi endpoint redirect", + "message": "Received endpoint redirect", "category": "blobset.blobs.receive", "timestamp": "2022-07-13T12:00:08.000Z", "level": 100 From cce99b62caa9fd00a0fe130171adbdb998df7f80 Mon Sep 17 00:00:00 2001 From: John R Date: Wed, 20 Jul 2022 11:45:44 -0400 Subject: [PATCH 37/57] Delete blobset_base64_error.json --- tests/state.tests/blobset_base64_error.json | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 tests/state.tests/blobset_base64_error.json diff --git a/tests/state.tests/blobset_base64_error.json b/tests/state.tests/blobset_base64_error.json deleted file mode 100644 index dce5d8be4d..0000000000 --- a/tests/state.tests/blobset_base64_error.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "version": 1, - "timestamp": "2022-07-13T12:00:02.000Z", - "blobset": { - "blobs": { - "_endpoint_redirect": { - "phase": "error", - "base64": "base64 encoded error message from base64 decoder" - } - } - } -} From d36c1ab32f5de0bdc013e30d2652e4e5d24f52bd Mon Sep 17 00:00:00 2001 From: John R Date: Wed, 20 Jul 2022 11:45:58 -0400 Subject: [PATCH 38/57] Delete blobset_receive.json --- tests/state.tests/blobset_receive.json | 44 -------------------------- 1 file changed, 44 deletions(-) delete mode 100644 tests/state.tests/blobset_receive.json diff --git a/tests/state.tests/blobset_receive.json b/tests/state.tests/blobset_receive.json deleted file mode 100644 index 5d821208dd..0000000000 --- a/tests/state.tests/blobset_receive.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "version": 1, - "timestamp": "2022-07-13T12:00:10.000Z", - "system": { - "hardware": { - "make": "ACME", - "model": "Bird Trap" - }, - "software": { - "firmware": "1.0" - }, - "serial_no": "000000", - "last_config": "2022-07-11T00:00:10.000Z", - "operational": true - }, - "blobset": { - "blobs": { - "arbitrary_manufacturer_id": { - "status": { - "message": "Received arbitrary_manufacturer_id", - "category": "blobset.blobs.receive", - "timestamp": "2022-07-13T12:00:08.000Z", - "level": 100 - } - }, - "_firmware_update": { - "status": { - "message": "Received firmware update", - "category": "blobset.blobs.receive", - "timestamp": "2022-07-13T12:00:08.000Z", - "level": 100 - } - }, - "_endpoint_redirect": { - "status": { - "message": "Received endpoint redirect", - "category": "blobset.blobs.receive", - "timestamp": "2022-07-13T12:00:08.000Z", - "level": 100 - } - } - } - } -} From 7be3f5a8343abbd6952efb0badae7f95b63231a6 Mon Sep 17 00:00:00 2001 From: John Randolph Date: Wed, 20 Jul 2022 11:47:05 -0400 Subject: [PATCH 39/57] Re-commit --- tests/state.tests/blobset_receive.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/state.tests/blobset_receive.json b/tests/state.tests/blobset_receive.json index 5d821208dd..a60cd41a7e 100644 --- a/tests/state.tests/blobset_receive.json +++ b/tests/state.tests/blobset_receive.json @@ -7,7 +7,7 @@ "model": "Bird Trap" }, "software": { - "firmware": "1.0" + "firmware": "1.1" }, "serial_no": "000000", "last_config": "2022-07-11T00:00:10.000Z", From cf67d64c506cefe13f1f994b8485e5a8e8fa1a90 Mon Sep 17 00:00:00 2001 From: John Randolph Date: Wed, 20 Jul 2022 11:49:56 -0400 Subject: [PATCH 40/57] Revert "Delete blobset_receive.json" This reverts commit d36c1ab32f5de0bdc013e30d2652e4e5d24f52bd. --- tests/state.tests/blobset_receive.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/state.tests/blobset_receive.json b/tests/state.tests/blobset_receive.json index a60cd41a7e..5b4c68b669 100644 --- a/tests/state.tests/blobset_receive.json +++ b/tests/state.tests/blobset_receive.json @@ -7,7 +7,11 @@ "model": "Bird Trap" }, "software": { +<<<<<<< HEAD "firmware": "1.1" +======= + "firmware": "1.0" +>>>>>>> parent of d36c1ab3 (Delete blobset_receive.json) }, "serial_no": "000000", "last_config": "2022-07-11T00:00:10.000Z", From d4f579305d79b65152cb33bc1bca348972828724 Mon Sep 17 00:00:00 2001 From: John R Date: Wed, 20 Jul 2022 11:55:29 -0400 Subject: [PATCH 41/57] Delete blobset_url_error.json --- tests/state.tests/blobset_url_error.json | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 tests/state.tests/blobset_url_error.json diff --git a/tests/state.tests/blobset_url_error.json b/tests/state.tests/blobset_url_error.json deleted file mode 100644 index c8424e5b73..0000000000 --- a/tests/state.tests/blobset_url_error.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "version": 1, - "timestamp": "2022-07-13T12:00:02.000Z", - "blobset": { - "blobs": { - "_firmware_update": { - "phase": "error", - "url": "https://manuf.com/path/file.zip", - "base64" "base64 encoded HTTPS error response message" - } - } - } -} From 876e9a0aef3baa20078e40f7c9058be8aba76f6c Mon Sep 17 00:00:00 2001 From: John Randolph Date: Wed, 20 Jul 2022 11:55:37 -0400 Subject: [PATCH 42/57] Add it back --- tests/state.tests/blobset_receive.json | 44 ++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 tests/state.tests/blobset_receive.json diff --git a/tests/state.tests/blobset_receive.json b/tests/state.tests/blobset_receive.json new file mode 100644 index 0000000000..a60cd41a7e --- /dev/null +++ b/tests/state.tests/blobset_receive.json @@ -0,0 +1,44 @@ +{ + "version": 1, + "timestamp": "2022-07-13T12:00:10.000Z", + "system": { + "hardware": { + "make": "ACME", + "model": "Bird Trap" + }, + "software": { + "firmware": "1.1" + }, + "serial_no": "000000", + "last_config": "2022-07-11T00:00:10.000Z", + "operational": true + }, + "blobset": { + "blobs": { + "arbitrary_manufacturer_id": { + "status": { + "message": "Received arbitrary_manufacturer_id", + "category": "blobset.blobs.receive", + "timestamp": "2022-07-13T12:00:08.000Z", + "level": 100 + } + }, + "_firmware_update": { + "status": { + "message": "Received firmware update", + "category": "blobset.blobs.receive", + "timestamp": "2022-07-13T12:00:08.000Z", + "level": 100 + } + }, + "_endpoint_redirect": { + "status": { + "message": "Received endpoint redirect", + "category": "blobset.blobs.receive", + "timestamp": "2022-07-13T12:00:08.000Z", + "level": 100 + } + } + } + } +} From 13eb9360e40ad224399fcec501f37bb240cade04 Mon Sep 17 00:00:00 2001 From: John Randolph Date: Wed, 20 Jul 2022 11:58:04 -0400 Subject: [PATCH 43/57] Again --- tests/state.tests/blobset_receive.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/state.tests/blobset_receive.json b/tests/state.tests/blobset_receive.json index a60cd41a7e..79fde36161 100644 --- a/tests/state.tests/blobset_receive.json +++ b/tests/state.tests/blobset_receive.json @@ -7,7 +7,7 @@ "model": "Bird Trap" }, "software": { - "firmware": "1.1" + "firmware": "1.2" }, "serial_no": "000000", "last_config": "2022-07-11T00:00:10.000Z", From af5d9b7d107714439ff15b3bd962f18802abdc8d Mon Sep 17 00:00:00 2001 From: John Randolph Date: Wed, 20 Jul 2022 15:19:25 -0400 Subject: [PATCH 44/57] Remove plural --- tests/state.tests/blobset_receive.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/state.tests/blobset_receive.json b/tests/state.tests/blobset_receive.json index 79fde36161..d85d5743b1 100644 --- a/tests/state.tests/blobset_receive.json +++ b/tests/state.tests/blobset_receive.json @@ -18,7 +18,7 @@ "arbitrary_manufacturer_id": { "status": { "message": "Received arbitrary_manufacturer_id", - "category": "blobset.blobs.receive", + "category": "blobset.blob.receive", "timestamp": "2022-07-13T12:00:08.000Z", "level": 100 } @@ -26,7 +26,7 @@ "_firmware_update": { "status": { "message": "Received firmware update", - "category": "blobset.blobs.receive", + "category": "blobset.blob.receive", "timestamp": "2022-07-13T12:00:08.000Z", "level": 100 } @@ -34,7 +34,7 @@ "_endpoint_redirect": { "status": { "message": "Received endpoint redirect", - "category": "blobset.blobs.receive", + "category": "blobset.blob.receive", "timestamp": "2022-07-13T12:00:08.000Z", "level": 100 } From f0bc914bd07d516b38bedc2e011bf34e5b13f7d5 Mon Sep 17 00:00:00 2001 From: John Randolph Date: Thu, 21 Jul 2022 12:45:16 -0400 Subject: [PATCH 45/57] Add schema update --- schema/state_blobset_blob.json | 6 ------ 1 file changed, 6 deletions(-) diff --git a/schema/state_blobset_blob.json b/schema/state_blobset_blob.json index b8120ec07d..658e04907a 100644 --- a/schema/state_blobset_blob.json +++ b/schema/state_blobset_blob.json @@ -4,12 +4,6 @@ "$schema": "http://json-schema.org/draft-07/schema#", "additionalProperties": false, "properties": { - "stage": { - "type": "string" - }, - "result": { - "type": "string" - }, "status": { "$ref": "file:common.json#/definitions/entry" } From 8f4fc952efa36fb22d29af6dec8b6e8c8abd9e93 Mon Sep 17 00:00:00 2001 From: John Randolph Date: Thu, 21 Jul 2022 13:39:58 -0400 Subject: [PATCH 46/57] Add _ to blobset{ "_key" } regex --- schema/state_blobset.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schema/state_blobset.json b/schema/state_blobset.json index e74e503032..f93ed9f9bb 100644 --- a/schema/state_blobset.json +++ b/schema/state_blobset.json @@ -8,7 +8,7 @@ "additionalProperties": false, "existingJavaType": "java.util.HashMap", "patternProperties": { - "^[a-z][a-z0-9]*(_[a-z0-9]+)*$": { + "^(_?)[a-z][a-z0-9]*(_[a-z0-9]+)*$": { "$ref": "file:state_blobset_blob.json#" } } From 094c0a2d7c1c867a82fd6d97d664851316fe59cb Mon Sep 17 00:00:00 2001 From: John Randolph Date: Thu, 21 Jul 2022 14:14:49 -0400 Subject: [PATCH 47/57] Add --- schema/config_blobset.json | 2 +- schema/config_blobset_blob.json | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/schema/config_blobset.json b/schema/config_blobset.json index a45eb3aff5..fc39c685f8 100644 --- a/schema/config_blobset.json +++ b/schema/config_blobset.json @@ -8,7 +8,7 @@ "additionalProperties": false, "existingJavaType": "java.util.HashMap", "patternProperties": { - "^[a-z][a-z0-9]*(_[a-z0-9]+)*$": { + "^(_?)[a-z][a-z0-9]*(_[a-z0-9]+)*$": { "$ref": "file:config_blobset_blob.json#" } } diff --git a/schema/config_blobset_blob.json b/schema/config_blobset_blob.json index c2aecbfa8a..0ddee6fbc3 100644 --- a/schema/config_blobset_blob.json +++ b/schema/config_blobset_blob.json @@ -4,10 +4,19 @@ "$schema": "http://json-schema.org/draft-07/schema#", "additionalProperties": false, "properties": { - "stage": { + "phase": { "type": "string" }, - "target": { + "content_type": { + "type": "string" + }, + "base64": { + "type": "string" + }, + "url": { + "type": "string" + }, + "sha256": { "type": "string" } } From e8846cc9737165305b518558225d2debdb7c0a46 Mon Sep 17 00:00:00 2001 From: John Randolph Date: Fri, 22 Jul 2022 13:29:20 -0400 Subject: [PATCH 48/57] Add "expected reference file"s --- tests/config.tests/applied.out | 0 tests/state.tests/blobset_receive.out | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/config.tests/applied.out create mode 100644 tests/state.tests/blobset_receive.out diff --git a/tests/config.tests/applied.out b/tests/config.tests/applied.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/state.tests/blobset_receive.out b/tests/state.tests/blobset_receive.out new file mode 100644 index 0000000000..e69de29bb2 From a01aab389c25ef7504b1c90ce016ad0103ab5367 Mon Sep 17 00:00:00 2001 From: John Randolph Date: Fri, 22 Jul 2022 14:24:50 -0400 Subject: [PATCH 49/57] Add gencode --- .gencode_hash.txt | 16 +- gencode/docs/config.html | 198 ++++++++++++++++-- gencode/docs/state.html | 152 +------------- .../java/udmi/schema/BlobBlobsetConfig.java | 30 ++- .../java/udmi/schema/BlobBlobsetState.java | 10 +- gencode/java/udmi/schema/BlobsetState.java | 6 +- .../python/udmi/schema/config_blobset_blob.py | 28 ++- gencode/python/udmi/schema/state_blobset.py | 4 - .../python/udmi/schema/state_blobset_blob.py | 8 - 9 files changed, 233 insertions(+), 219 deletions(-) diff --git a/.gencode_hash.txt b/.gencode_hash.txt index fc2f874d6b..30d12f52ee 100644 --- a/.gencode_hash.txt +++ b/.gencode_hash.txt @@ -1,4 +1,4 @@ -23329e12348400aa5e3fd08cdd0e973ac4783421897cf2b97a48df65a7f209fe gencode/docs/config.html +89703da5452a2976b2c48bd3362a1d95ad323528e3a3b3b675ad17272c78e17c gencode/docs/config.html e2944b13db5ff06be9caea51d03bca48f2cb093a8bb583dca14051255d34ea6b gencode/docs/envelope.html fc00c1f0aaa73dd31b6b4214433e47a63bf787a1796c7ff54b60a04dc7a3ace4 gencode/docs/event_discovery.html 8133e380e40f27c56accbffc665b2eeb56ec84a4da3b52ba7aa5e439c9c40572 gencode/docs/event_pointset.html @@ -8,14 +8,14 @@ a82821e72af6d0ee35800e6262eb9bb05256309b98aed2dad1a368fd2d6882bb gencode/docs/i 741b880216be3743f6747800a042f2dbd89f3b0344c6b0a965f4bc010f03a930 gencode/docs/schema_doc.css 878ea88206c974f40643c3cc430875f9c4e8c5e3fd6bcd6358bd3eb6d48699a9 gencode/docs/schema_doc.min.js 7ed934930aee763e0beebc349725ba3909115e8d346bb762f28bcbe745bb163a gencode/docs/schema_extras.js -5a1e8ec9c93f3091b3c8a81b0b804b05cc333744e13efb6a379bd9c84326b9fc gencode/docs/state.html +44ef15041048512e83af20e0949c1b8a2e3925fd997903c69884586f46ac6f81 gencode/docs/state.html d39d7fe37a41c74a40080af7b0a429d201ab1fdff7444428c4b98eb7b38c332b gencode/java/udmi/schema/Asset.java 0edc9b905bbe4ad0ee5ffdb7fb5d65a00ad171b2dad299ef128e59c0bd9dddc0 gencode/java/udmi/schema/AuditEvent.java 0825a5cec83003bb0a6488c4ed7010a04ae0d3848ef36fe01bb4e6718ba7b96d gencode/java/udmi/schema/Aux.java -724ad8db7982f4d703c05245119dc7f7ff731d2141dd861893b55684e3cd6224 gencode/java/udmi/schema/BlobBlobsetConfig.java -7f54da38284a1010de1a2590381022fa16d27fd141d76d1ba6eb471de4d94781 gencode/java/udmi/schema/BlobBlobsetState.java +773223fd813648a9fb1eb890287f45fa127ef19a553186395f05a1a47d818716 gencode/java/udmi/schema/BlobBlobsetConfig.java +2c03651cb2ecda072b1418222eebb5560185669f8ffdd03021ad5ad8ff7ba3b0 gencode/java/udmi/schema/BlobBlobsetState.java d2c5b5aae8db27b68104fc83a1f38de0a3f1b5d683f2b13599adf24e96c7d124 gencode/java/udmi/schema/BlobsetConfig.java -b6ff9b8739a9c3bb6972f73db6fc54f451189c13b273e58bc11cb3d82c74ad40 gencode/java/udmi/schema/BlobsetState.java +fcbed49f1af8b791d8c52bcbe18f65521a79d9ac3eb33ec3afd9b342ab2bfc56 gencode/java/udmi/schema/BlobsetState.java a2eeff86f4302272736d84602e2ca36a64d27c8ef6761cc05ffb8ad17b030d4d gencode/java/udmi/schema/CloudModel.java ff79de9390aa25bb45fb3e2ebb682c865ccab764f56d9644377d9d28c0ab10e4 gencode/java/udmi/schema/Config.java 587d67a67431349939dffd37b880c44e798a1eb607d54bd6d8a077bbac668067 gencode/java/udmi/schema/DiscoveryCommand.java @@ -72,7 +72,7 @@ d3968b92497e83a63f18cc0e74484a9807f1bb92db0c92d556ec2caaa143d645 gencode/java/u 704c8f0eec0b87015af8f7e524375f651b3d35f659ec89b4b022f8c1d0813ec5 gencode/python/udmi/schema/common.py b975892df78076dabc797b4c0be87f20b33eacda11f9d1ac1c09be33d4937a87 gencode/python/udmi/schema/config.py 191e1926c16b55f4ef350a711f540eef17a0ec60bec8c193c94182786dc3624b gencode/python/udmi/schema/config_blobset.py -7abd40beb86ebd7eaeaacdeba895b95563b79b651ad2cfd73bdcac865c90a3a9 gencode/python/udmi/schema/config_blobset_blob.py +95125e3459cec6e12f4e0d3912f9c519b1aec41cf1df9720b54481e69632d7f4 gencode/python/udmi/schema/config_blobset_blob.py ec6c6ab1fb0f37a29b7ebd162aa77da7f1e261e80da376942a3b39d17ccf1be4 gencode/python/udmi/schema/config_discovery.py a5edb9ac5ecd5a4459f93ce613691735f299f35718f2e35410206fc91c263dd1 gencode/python/udmi/schema/config_discovery_family.py b461bdc24310ef972faf579b5be577b5af67fb0977d6afb4c42955211b26e3d5 gencode/python/udmi/schema/config_gateway.py @@ -106,8 +106,8 @@ a58f8c98e837a5b56126ca0f410e02f1e9cfcd80a8cb429e0ef522defab1f690 gencode/python 1f521678016ad267ad1c817896c7900ba30a85fc694669577b71148193db32c1 gencode/python/udmi/schema/reflect_config.py 1a4eef286957d77418777eb7f4cd1ee13aa24c2d916ec71a7ff4e56de76e303e gencode/python/udmi/schema/reflect_state.py 32bc70a30e37e89cfae14b44add18d546a6f9e00a3ec3519ede9c7486114d39c gencode/python/udmi/schema/state.py -c8c8ecae303d9c96fb7a97106d722b32db8c3728a44a46db028cf0376d9dc79f gencode/python/udmi/schema/state_blobset.py -a5a914cb5d74c29671a4d29dfa6c700b3fec27d695d607e28814ad31307e82da gencode/python/udmi/schema/state_blobset_blob.py +4a908cee3fb8afb559bcbfa594e57dbc515a35e4468e02600751b2fcce05a238 gencode/python/udmi/schema/state_blobset.py +16a3541464d48b0cda007668cbc9c00911b94d044fe69a23773df2648359723f gencode/python/udmi/schema/state_blobset_blob.py 26443a1f6d0be3469ff93aa7fdb4e6682e0439a3b29a8e237998dcebec5f6901 gencode/python/udmi/schema/state_discovery.py 187400078dfc89912062ca1ad92f61e32d28126ae56119d83e6767d58cda1117 gencode/python/udmi/schema/state_discovery_family.py 05e82aa15c64842e206ae8ce3d5810d115bb890d009ea5d657822fad0e0d2165 gencode/python/udmi/schema/state_gateway.py diff --git a/gencode/docs/config.html b/gencode/docs/config.html index 910f942c4f..bc60a67d97 100644 --- a/gencode/docs/config.html +++ b/gencode/docs/config.html @@ -1190,7 +1190,7 @@

+ aria-expanded="" aria-controls="blobset_blobs_pattern1" onclick="setAnchor('#blobset_blobs_pattern1')">^(_?)[a-z][a-z0-9]*(_[a-z0-9]+)*$ Pattern Property

@@ -1201,7 +1201,7 @@

All property whose name matches the following regular expression must respect the following conditions

- Property name regular expression: ^[a-z][a-z0-9]*(_[a-z0-9]+)*$ + Property name regular expression: ^(_?)[a-z][a-z0-9]*(_[a-z0-9]+)*$
root @@ -1235,18 +1235,180 @@

-
+
+
+
+

+ +

+
+ +
+
+ +
root + + + + blobset + + + + blobs + + + + Blob Blobset Config + + + + phase
Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
root + + + + blobset + + + + blobs + + + + Blob Blobset Config + + + + content_type
Type: string
+ + + + + + + +
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
root + + + + blobset + + + + blobs + + + + Blob Blobset Config + + + + base64
Type: string
+ + + + + + + +
+
+
+
+
-
+

- +

-
+
root @@ -1277,7 +1439,7 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - stage

Type: string
+ url
Type: string
@@ -1289,18 +1451,18 @@

-
+
-
+

- +

-
+
root @@ -1331,7 +1493,7 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - target

Type: string
+ sha256
Type: string
diff --git a/gencode/docs/state.html b/gencode/docs/state.html index f9bb83b92b..3bbc6ee31b 100644 --- a/gencode/docs/state.html +++ b/gencode/docs/state.html @@ -2748,46 +2748,6 @@

-
-
-
-

- -

-
- -
-
- -
root - - - - blobset - - - - state_etag
Type: string
- - - - -

Must be at most 32 characters long

- - -
-
-
-
@@ -2829,7 +2789,7 @@

+ aria-expanded="" aria-controls="blobset_blobs_pattern1" onclick="setAnchor('#blobset_blobs_pattern1')">^(_?)[a-z][a-z0-9]*(_[a-z0-9]+)*$ Pattern Property

@@ -2840,7 +2800,7 @@

All property whose name matches the following regular expression must respect the following conditions

- Property name regular expression: ^[a-z][a-z0-9]*(_[a-z0-9]+)*$ + Property name regular expression: ^(_?)[a-z][a-z0-9]*(_[a-z0-9]+)*$
root @@ -2874,114 +2834,6 @@

-
-
-
-

- -

-
- -
-
- -
root - - - - blobset - - - - blobs - - - - Blob Blobset State - - - - stage
Type: string
- - - - - - - -
-
-
-
-
-
-
-

- -

-
- -
-
- -
root - - - - blobset - - - - blobs - - - - Blob Blobset State - - - - result
Type: string
- - - - - - - -
-
-
-
diff --git a/gencode/java/udmi/schema/BlobBlobsetConfig.java b/gencode/java/udmi/schema/BlobBlobsetConfig.java index e5cc4e7ffd..6d9e857fe5 100644 --- a/gencode/java/udmi/schema/BlobBlobsetConfig.java +++ b/gencode/java/udmi/schema/BlobBlobsetConfig.java @@ -15,22 +15,34 @@ */ @JsonInclude(JsonInclude.Include.NON_NULL) @JsonPropertyOrder({ - "stage", - "target" + "phase", + "content_type", + "base64", + "url", + "sha256" }) @Generated("jsonschema2pojo") public class BlobBlobsetConfig { - @JsonProperty("stage") - public String stage; - @JsonProperty("target") - public String target; + @JsonProperty("phase") + public String phase; + @JsonProperty("content_type") + public String content_type; + @JsonProperty("base64") + public String base64; + @JsonProperty("url") + public String url; + @JsonProperty("sha256") + public String sha256; @Override public int hashCode() { int result = 1; - result = ((result* 31)+((this.stage == null)? 0 :this.stage.hashCode())); - result = ((result* 31)+((this.target == null)? 0 :this.target.hashCode())); + result = ((result* 31)+((this.phase == null)? 0 :this.phase.hashCode())); + result = ((result* 31)+((this.base64 == null)? 0 :this.base64 .hashCode())); + result = ((result* 31)+((this.content_type == null)? 0 :this.content_type.hashCode())); + result = ((result* 31)+((this.sha256 == null)? 0 :this.sha256 .hashCode())); + result = ((result* 31)+((this.url == null)? 0 :this.url.hashCode())); return result; } @@ -43,7 +55,7 @@ public boolean equals(Object other) { return false; } BlobBlobsetConfig rhs = ((BlobBlobsetConfig) other); - return (((this.stage == rhs.stage)||((this.stage!= null)&&this.stage.equals(rhs.stage)))&&((this.target == rhs.target)||((this.target!= null)&&this.target.equals(rhs.target)))); + return ((((((this.phase == rhs.phase)||((this.phase!= null)&&this.phase.equals(rhs.phase)))&&((this.base64 == rhs.base64)||((this.base64 != null)&&this.base64 .equals(rhs.base64))))&&((this.content_type == rhs.content_type)||((this.content_type!= null)&&this.content_type.equals(rhs.content_type))))&&((this.sha256 == rhs.sha256)||((this.sha256 != null)&&this.sha256 .equals(rhs.sha256))))&&((this.url == rhs.url)||((this.url!= null)&&this.url.equals(rhs.url)))); } } diff --git a/gencode/java/udmi/schema/BlobBlobsetState.java b/gencode/java/udmi/schema/BlobBlobsetState.java index ac8b771f2e..e8e80893a1 100644 --- a/gencode/java/udmi/schema/BlobBlobsetState.java +++ b/gencode/java/udmi/schema/BlobBlobsetState.java @@ -15,17 +15,11 @@ */ @JsonInclude(JsonInclude.Include.NON_NULL) @JsonPropertyOrder({ - "stage", - "result", "status" }) @Generated("jsonschema2pojo") public class BlobBlobsetState { - @JsonProperty("stage") - public String stage; - @JsonProperty("result") - public String result; /** * Entry *

@@ -38,8 +32,6 @@ public class BlobBlobsetState { @Override public int hashCode() { int result = 1; - result = ((result* 31)+((this.result == null)? 0 :this.result.hashCode())); - result = ((result* 31)+((this.stage == null)? 0 :this.stage.hashCode())); result = ((result* 31)+((this.status == null)? 0 :this.status.hashCode())); return result; } @@ -53,7 +45,7 @@ public boolean equals(Object other) { return false; } BlobBlobsetState rhs = ((BlobBlobsetState) other); - return ((((this.result == rhs.result)||((this.result!= null)&&this.result.equals(rhs.result)))&&((this.stage == rhs.stage)||((this.stage!= null)&&this.stage.equals(rhs.stage))))&&((this.status == rhs.status)||((this.status!= null)&&this.status.equals(rhs.status)))); + return ((this.status == rhs.status)||((this.status!= null)&&this.status.equals(rhs.status))); } } diff --git a/gencode/java/udmi/schema/BlobsetState.java b/gencode/java/udmi/schema/BlobsetState.java index c927698abc..2c6f0bee63 100644 --- a/gencode/java/udmi/schema/BlobsetState.java +++ b/gencode/java/udmi/schema/BlobsetState.java @@ -16,14 +16,11 @@ */ @JsonInclude(JsonInclude.Include.NON_NULL) @JsonPropertyOrder({ - "state_etag", "blobs" }) @Generated("jsonschema2pojo") public class BlobsetState { - @JsonProperty("state_etag") - public java.lang.String state_etag; /** * * (Required) @@ -36,7 +33,6 @@ public class BlobsetState { public int hashCode() { int result = 1; result = ((result* 31)+((this.blobs == null)? 0 :this.blobs.hashCode())); - result = ((result* 31)+((this.state_etag == null)? 0 :this.state_etag.hashCode())); return result; } @@ -49,7 +45,7 @@ public boolean equals(Object other) { return false; } BlobsetState rhs = ((BlobsetState) other); - return (((this.blobs == rhs.blobs)||((this.blobs!= null)&&this.blobs.equals(rhs.blobs)))&&((this.state_etag == rhs.state_etag)||((this.state_etag!= null)&&this.state_etag.equals(rhs.state_etag)))); + return ((this.blobs == rhs.blobs)||((this.blobs!= null)&&this.blobs.equals(rhs.blobs))); } } diff --git a/gencode/python/udmi/schema/config_blobset_blob.py b/gencode/python/udmi/schema/config_blobset_blob.py index 5a4a3f29c1..b8fbb4ce55 100644 --- a/gencode/python/udmi/schema/config_blobset_blob.py +++ b/gencode/python/udmi/schema/config_blobset_blob.py @@ -5,16 +5,22 @@ class BlobBlobsetConfig: """Generated schema class""" def __init__(self): - self.stage = None - self.target = None + self.phase = None + self.content_type = None + self.base64 = None + self.url = None + self.sha256 = None @staticmethod def from_dict(source): if not source: return None result = BlobBlobsetConfig() - result.stage = source.get('stage') - result.target = source.get('target') + result.phase = source.get('phase') + result.content_type = source.get('content_type') + result.base64 = source.get('base64') + result.url = source.get('url') + result.sha256 = source.get('sha256') return result @staticmethod @@ -35,8 +41,14 @@ def expand_dict(input): def to_dict(self): result = {} - if self.stage: - result['stage'] = self.stage # 5 - if self.target: - result['target'] = self.target # 5 + if self.phase: + result['phase'] = self.phase # 5 + if self.content_type: + result['content_type'] = self.content_type # 5 + if self.base64: + result['base64'] = self.base64 # 5 + if self.url: + result['url'] = self.url # 5 + if self.sha256: + result['sha256'] = self.sha256 # 5 return result diff --git a/gencode/python/udmi/schema/state_blobset.py b/gencode/python/udmi/schema/state_blobset.py index 442abc89a5..9b0aae0171 100644 --- a/gencode/python/udmi/schema/state_blobset.py +++ b/gencode/python/udmi/schema/state_blobset.py @@ -6,7 +6,6 @@ class BlobsetState: """Generated schema class""" def __init__(self): - self.state_etag = None self.blobs = None @staticmethod @@ -14,7 +13,6 @@ def from_dict(source): if not source: return None result = BlobsetState() - result.state_etag = source.get('state_etag') result.blobs = BlobBlobsetState.map_from(source.get('blobs')) return result @@ -36,8 +34,6 @@ def expand_dict(input): def to_dict(self): result = {} - if self.state_etag: - result['state_etag'] = self.state_etag # 5 if self.blobs: result['blobs'] = BlobBlobsetState.expand_dict(self.blobs) # 2 return result diff --git a/gencode/python/udmi/schema/state_blobset_blob.py b/gencode/python/udmi/schema/state_blobset_blob.py index f6b089f5ff..0ffe9a9f9e 100644 --- a/gencode/python/udmi/schema/state_blobset_blob.py +++ b/gencode/python/udmi/schema/state_blobset_blob.py @@ -6,8 +6,6 @@ class BlobBlobsetState: """Generated schema class""" def __init__(self): - self.stage = None - self.result = None self.status = None @staticmethod @@ -15,8 +13,6 @@ def from_dict(source): if not source: return None result = BlobBlobsetState() - result.stage = source.get('stage') - result.result = source.get('result') result.status = Entry.from_dict(source.get('status')) return result @@ -38,10 +34,6 @@ def expand_dict(input): def to_dict(self): result = {} - if self.stage: - result['stage'] = self.stage # 5 - if self.result: - result['result'] = self.result # 5 if self.status: result['status'] = self.status.to_dict() # 4 return result From 1f3e236ccb49141e92415bb6ba3e9f4fd20f8fb8 Mon Sep 17 00:00:00 2001 From: John Randolph Date: Fri, 22 Jul 2022 15:49:02 -0400 Subject: [PATCH 50/57] Minimal regex --- .gencode_hash.txt | 4 ++-- gencode/docs/config.html | 4 ++-- gencode/docs/state.html | 4 ++-- schema/config_blobset.json | 2 +- schema/state_blobset.json | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.gencode_hash.txt b/.gencode_hash.txt index 30d12f52ee..b84c1859cd 100644 --- a/.gencode_hash.txt +++ b/.gencode_hash.txt @@ -1,4 +1,4 @@ -89703da5452a2976b2c48bd3362a1d95ad323528e3a3b3b675ad17272c78e17c gencode/docs/config.html +8fe2bb5cc0e0307d51d452331460b50526a47938345d3810c3db9cadc6bf5e20 gencode/docs/config.html e2944b13db5ff06be9caea51d03bca48f2cb093a8bb583dca14051255d34ea6b gencode/docs/envelope.html fc00c1f0aaa73dd31b6b4214433e47a63bf787a1796c7ff54b60a04dc7a3ace4 gencode/docs/event_discovery.html 8133e380e40f27c56accbffc665b2eeb56ec84a4da3b52ba7aa5e439c9c40572 gencode/docs/event_pointset.html @@ -8,7 +8,7 @@ a82821e72af6d0ee35800e6262eb9bb05256309b98aed2dad1a368fd2d6882bb gencode/docs/i 741b880216be3743f6747800a042f2dbd89f3b0344c6b0a965f4bc010f03a930 gencode/docs/schema_doc.css 878ea88206c974f40643c3cc430875f9c4e8c5e3fd6bcd6358bd3eb6d48699a9 gencode/docs/schema_doc.min.js 7ed934930aee763e0beebc349725ba3909115e8d346bb762f28bcbe745bb163a gencode/docs/schema_extras.js -44ef15041048512e83af20e0949c1b8a2e3925fd997903c69884586f46ac6f81 gencode/docs/state.html +cbbff5d74adb59cd83bfb37813b008a9970c239b2fb1084dc8a020b8856eac01 gencode/docs/state.html d39d7fe37a41c74a40080af7b0a429d201ab1fdff7444428c4b98eb7b38c332b gencode/java/udmi/schema/Asset.java 0edc9b905bbe4ad0ee5ffdb7fb5d65a00ad171b2dad299ef128e59c0bd9dddc0 gencode/java/udmi/schema/AuditEvent.java 0825a5cec83003bb0a6488c4ed7010a04ae0d3848ef36fe01bb4e6718ba7b96d gencode/java/udmi/schema/Aux.java diff --git a/gencode/docs/config.html b/gencode/docs/config.html index bc60a67d97..047127faa4 100644 --- a/gencode/docs/config.html +++ b/gencode/docs/config.html @@ -1190,7 +1190,7 @@

+ aria-expanded="" aria-controls="blobset_blobs_pattern1" onclick="setAnchor('#blobset_blobs_pattern1')">^_?[a-z][a-z0-9]*(_[a-z0-9]+)*$ Pattern Property

@@ -1201,7 +1201,7 @@

All property whose name matches the following regular expression must respect the following conditions

- Property name regular expression: ^(_?)[a-z][a-z0-9]*(_[a-z0-9]+)*$ + Property name regular expression: ^_?[a-z][a-z0-9]*(_[a-z0-9]+)*$
root diff --git a/gencode/docs/state.html b/gencode/docs/state.html index 3bbc6ee31b..96b92c60cf 100644 --- a/gencode/docs/state.html +++ b/gencode/docs/state.html @@ -2789,7 +2789,7 @@

+ aria-expanded="" aria-controls="blobset_blobs_pattern1" onclick="setAnchor('#blobset_blobs_pattern1')">^_?[a-z][a-z0-9]*(_[a-z0-9]+)*$ Pattern Property

@@ -2800,7 +2800,7 @@

All property whose name matches the following regular expression must respect the following conditions

- Property name regular expression: ^(_?)[a-z][a-z0-9]*(_[a-z0-9]+)*$ + Property name regular expression: ^_?[a-z][a-z0-9]*(_[a-z0-9]+)*$
root diff --git a/schema/config_blobset.json b/schema/config_blobset.json index fc39c685f8..55e1478e44 100644 --- a/schema/config_blobset.json +++ b/schema/config_blobset.json @@ -8,7 +8,7 @@ "additionalProperties": false, "existingJavaType": "java.util.HashMap", "patternProperties": { - "^(_?)[a-z][a-z0-9]*(_[a-z0-9]+)*$": { + "^_?[a-z][a-z0-9]*(_[a-z0-9]+)*$": { "$ref": "file:config_blobset_blob.json#" } } diff --git a/schema/state_blobset.json b/schema/state_blobset.json index f93ed9f9bb..3fadfd5349 100644 --- a/schema/state_blobset.json +++ b/schema/state_blobset.json @@ -8,7 +8,7 @@ "additionalProperties": false, "existingJavaType": "java.util.HashMap", "patternProperties": { - "^(_?)[a-z][a-z0-9]*(_[a-z0-9]+)*$": { + "^_?[a-z][a-z0-9]*(_[a-z0-9]+)*$": { "$ref": "file:state_blobset_blob.json#" } } From 4a4d1bbf0338c166af50b3b7d03577bb0252613e Mon Sep 17 00:00:00 2001 From: John Randolph Date: Mon, 25 Jul 2022 11:06:17 -0400 Subject: [PATCH 51/57] Debug this a bit --- bin/check_version | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/bin/check_version b/bin/check_version index f574ceecfe..a076adb425 100755 --- a/bin/check_version +++ b/bin/check_version @@ -15,10 +15,14 @@ if [[ -z $origin ]]; then origin=faucetsdn fi -git fetch $origin --tags 2>/dev/null +git fetch $origin --tags # 2>/dev/null -upstream_ver=`git describe $origin/$branch --abbrev=0 || echo unknown` -current_ver=`git describe HEAD --abbrev=0 || echo unknown` +if [[ $? -ne 0 ]]; then + current_ver="unknown" +else + upstream_ver=`git describe $origin/$branch --abbrev=0 || echo unknown` + current_ver=`git describe HEAD --abbrev=0 || echo unknown` +fi if [[ $current_ver != $upstream_ver ]]; then echo From 89e2a2f48c1d5f42561e4482e140a990ab3706e6 Mon Sep 17 00:00:00 2001 From: John Randolph Date: Tue, 26 Jul 2022 11:03:33 -0400 Subject: [PATCH 52/57] Remove old test steady Add fail tests for incomplete url or payload --- schema/config_blobset_blob.json | 15 +++++++++++---- tests/config.tests/apply.out | 0 .../{applied.json => blobset_applied.json} | 0 .../{applied.out => blobset_applied.out} | 0 .../blobset_applied_incomplete_payload.json | 12 ++++++++++++ .../blobset_applied_incomplete_payload.out | 2 ++ .../blobset_applied_incomplete_url.json | 12 ++++++++++++ .../blobset_applied_incomplete_url.out | 2 ++ tests/config.tests/steady.json | 12 ------------ tests/config.tests/steady.out | 0 10 files changed, 39 insertions(+), 16 deletions(-) delete mode 100644 tests/config.tests/apply.out rename tests/config.tests/{applied.json => blobset_applied.json} (100%) rename tests/config.tests/{applied.out => blobset_applied.out} (100%) create mode 100644 tests/config.tests/blobset_applied_incomplete_payload.json create mode 100644 tests/config.tests/blobset_applied_incomplete_payload.out create mode 100644 tests/config.tests/blobset_applied_incomplete_url.json create mode 100644 tests/config.tests/blobset_applied_incomplete_url.out delete mode 100644 tests/config.tests/steady.json delete mode 100644 tests/config.tests/steady.out diff --git a/schema/config_blobset_blob.json b/schema/config_blobset_blob.json index 0ddee6fbc3..bc8801d315 100644 --- a/schema/config_blobset_blob.json +++ b/schema/config_blobset_blob.json @@ -11,13 +11,20 @@ "type": "string" }, "base64": { - "type": "string" + "type": "string", + "pattern": "^[0-9A-Za-z=+/]{4,}$" }, "url": { - "type": "string" + "type": "string", + "format": "uri" }, "sha256": { - "type": "string" + "type": "string", + "pattern": "^[0-9A-Za-z]{64}$" } - } + }, + "oneOf": [ + { "required": ["content_type", "base64"] }, + { "required": ["url", "sha256"] } + ] } diff --git a/tests/config.tests/apply.out b/tests/config.tests/apply.out deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/config.tests/applied.json b/tests/config.tests/blobset_applied.json similarity index 100% rename from tests/config.tests/applied.json rename to tests/config.tests/blobset_applied.json diff --git a/tests/config.tests/applied.out b/tests/config.tests/blobset_applied.out similarity index 100% rename from tests/config.tests/applied.out rename to tests/config.tests/blobset_applied.out diff --git a/tests/config.tests/blobset_applied_incomplete_payload.json b/tests/config.tests/blobset_applied_incomplete_payload.json new file mode 100644 index 0000000000..9e2cb036f2 --- /dev/null +++ b/tests/config.tests/blobset_applied_incomplete_payload.json @@ -0,0 +1,12 @@ +{ + "version": 1, + "blobset": { + "blobs": { + "arbitrary_manufacturer_id": { + "phase": "applied", + "base64": "UEsDBAoAAAAAAAZZ3VSU7ZiuBQAAAAUAAAAIABwAYm9vdC5pbWdVVAkAA9xqvGLcarxidXgLAAEEFjQAAARTXwEAYm9vdApQSwMECgAAAAAAB1ndVK9osDMDAAAAAwAAAAYAHABmdy5pbWdVVAkAA95qvGLearxidXgLAAEEFjQAAARTXwEAZncKUEsBAh4DCgAAAAAABlndVJTtmK4FAAAABQAAAAgAGAAAAAAAAQAAAKSBAAAAAGJvb3QuaW1nVVQFAAPcarxidXgLAAEEFjQAAARTXwEAUEsBAh4DCgAAAAAAB1ndVK9osDMDAAAAAwAAAAYAGAAAAAAAAQAAAKSBRwAAAGZ3LmltZ1VUBQAD3mq8YnV4CwABBBY0AAAEU18BAFBLBQYAAAAAAgACAJoAAACKAAAAAAA=" + } + } + }, + "timestamp": "2022-07-13T12:00:00.000Z" +} diff --git a/tests/config.tests/blobset_applied_incomplete_payload.out b/tests/config.tests/blobset_applied_incomplete_payload.out new file mode 100644 index 0000000000..5889b40947 --- /dev/null +++ b/tests/config.tests/blobset_applied_incomplete_payload.out @@ -0,0 +1,2 @@ +1 schema violations found + instance failed to match exactly one schema (matched 0 out of 2) diff --git a/tests/config.tests/blobset_applied_incomplete_url.json b/tests/config.tests/blobset_applied_incomplete_url.json new file mode 100644 index 0000000000..a0b950fcb2 --- /dev/null +++ b/tests/config.tests/blobset_applied_incomplete_url.json @@ -0,0 +1,12 @@ +{ + "version": 1, + "blobset": { + "blobs": { + "_firmware_update": { + "phase": "applied", + "url": "https://manuf.com/path/file.zip" + } + } + }, + "timestamp": "2022-07-13T12:00:00.000Z" +} diff --git a/tests/config.tests/blobset_applied_incomplete_url.out b/tests/config.tests/blobset_applied_incomplete_url.out new file mode 100644 index 0000000000..5889b40947 --- /dev/null +++ b/tests/config.tests/blobset_applied_incomplete_url.out @@ -0,0 +1,2 @@ +1 schema violations found + instance failed to match exactly one schema (matched 0 out of 2) diff --git a/tests/config.tests/steady.json b/tests/config.tests/steady.json deleted file mode 100644 index 7b4e43a7f4..0000000000 --- a/tests/config.tests/steady.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "version": 1, - "blobset": { - "blobs": { - "codebase": { - "stage": null, - "target": null - } - } - }, - "timestamp": "2021-06-17T12:20:44.254Z" -} diff --git a/tests/config.tests/steady.out b/tests/config.tests/steady.out deleted file mode 100644 index e69de29bb2..0000000000 From 84939e1a4c709392bfe1e5a1707aa924b5016cd2 Mon Sep 17 00:00:00 2001 From: John Randolph Date: Tue, 26 Jul 2022 11:07:06 -0400 Subject: [PATCH 53/57] gencode updates --- .gencode_hash.txt | 4 +- gencode/docs/config.html | 119 +++++++++++++++++- .../java/udmi/schema/BlobBlobsetConfig.java | 3 +- 3 files changed, 120 insertions(+), 6 deletions(-) diff --git a/.gencode_hash.txt b/.gencode_hash.txt index b84c1859cd..b8bd5921d3 100644 --- a/.gencode_hash.txt +++ b/.gencode_hash.txt @@ -1,4 +1,4 @@ -8fe2bb5cc0e0307d51d452331460b50526a47938345d3810c3db9cadc6bf5e20 gencode/docs/config.html +3e0762706ab532a69da44dc7cc562aa050533c3df4da222a82199ced7742c5fa gencode/docs/config.html e2944b13db5ff06be9caea51d03bca48f2cb093a8bb583dca14051255d34ea6b gencode/docs/envelope.html fc00c1f0aaa73dd31b6b4214433e47a63bf787a1796c7ff54b60a04dc7a3ace4 gencode/docs/event_discovery.html 8133e380e40f27c56accbffc665b2eeb56ec84a4da3b52ba7aa5e439c9c40572 gencode/docs/event_pointset.html @@ -12,7 +12,7 @@ cbbff5d74adb59cd83bfb37813b008a9970c239b2fb1084dc8a020b8856eac01 gencode/docs/s d39d7fe37a41c74a40080af7b0a429d201ab1fdff7444428c4b98eb7b38c332b gencode/java/udmi/schema/Asset.java 0edc9b905bbe4ad0ee5ffdb7fb5d65a00ad171b2dad299ef128e59c0bd9dddc0 gencode/java/udmi/schema/AuditEvent.java 0825a5cec83003bb0a6488c4ed7010a04ae0d3848ef36fe01bb4e6718ba7b96d gencode/java/udmi/schema/Aux.java -773223fd813648a9fb1eb890287f45fa127ef19a553186395f05a1a47d818716 gencode/java/udmi/schema/BlobBlobsetConfig.java +d5adb804697243f97cdd8589750401654f3fab075a9aeac4f2851e46695ef11d gencode/java/udmi/schema/BlobBlobsetConfig.java 2c03651cb2ecda072b1418222eebb5560185669f8ffdd03021ad5ad8ff7ba3b0 gencode/java/udmi/schema/BlobBlobsetState.java d2c5b5aae8db27b68104fc83a1f38de0a3f1b5d683f2b13599adf24e96c7d124 gencode/java/udmi/schema/BlobsetConfig.java fcbed49f1af8b791d8c52bcbe18f65521a79d9ac3eb33ec3afd9b342ab2bfc56 gencode/java/udmi/schema/BlobsetState.java diff --git a/gencode/docs/config.html b/gencode/docs/config.html index 047127faa4..3401fc2586 100644 --- a/gencode/docs/config.html +++ b/gencode/docs/config.html @@ -1228,7 +1228,120 @@

config_blobset_blob.json#

Type: object
- No Additional Properties + No Additional Properties

+ +

+
+ + +
root + + + + blobset + + + + blobs + + + + Blob Blobset Config + + + + oneOf + + + + item 0
Type: object
+ +
+

The following properties are required:

+
  • content_type
  • base64
+
+ + + + + +
+ + +
root + + + + blobset + + + + blobs + + + + Blob Blobset Config + + + + oneOf + + + + item 1
Type: object
+ +
+

The following properties are required:

+
  • url
  • sha256
+
+ + + + + +
@@ -1386,7 +1499,7 @@

/> base64

Type: string
- +Must match regular expression: ^[0-9A-Za-z=+/]{4,}$ @@ -1494,7 +1607,7 @@

/> sha256

Type: string
- +Must match regular expression: ^[0-9A-Za-z]{64}$ diff --git a/gencode/java/udmi/schema/BlobBlobsetConfig.java b/gencode/java/udmi/schema/BlobBlobsetConfig.java index 6d9e857fe5..de0fa2e0a7 100644 --- a/gencode/java/udmi/schema/BlobBlobsetConfig.java +++ b/gencode/java/udmi/schema/BlobBlobsetConfig.java @@ -1,6 +1,7 @@ package udmi.schema; +import java.net.URI; import javax.annotation.processing.Generated; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; @@ -31,7 +32,7 @@ public class BlobBlobsetConfig { @JsonProperty("base64") public String base64; @JsonProperty("url") - public String url; + public URI url; @JsonProperty("sha256") public String sha256; From 9a40d26bdac8f14b68950efbc2a6de6203b70f20 Mon Sep 17 00:00:00 2001 From: John Randolph Date: Tue, 26 Jul 2022 17:05:27 -0400 Subject: [PATCH 54/57] Hash difference? --- .gencode_hash.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gencode_hash.txt b/.gencode_hash.txt index b8bd5921d3..125eb8ae9a 100644 --- a/.gencode_hash.txt +++ b/.gencode_hash.txt @@ -1,4 +1,4 @@ -3e0762706ab532a69da44dc7cc562aa050533c3df4da222a82199ced7742c5fa gencode/docs/config.html +8c0f20d1da08192edceb6c0c3d3b6de85a2cc5abd5d8188fc56000605871de7b gencode/docs/config.html e2944b13db5ff06be9caea51d03bca48f2cb093a8bb583dca14051255d34ea6b gencode/docs/envelope.html fc00c1f0aaa73dd31b6b4214433e47a63bf787a1796c7ff54b60a04dc7a3ace4 gencode/docs/event_discovery.html 8133e380e40f27c56accbffc665b2eeb56ec84a4da3b52ba7aa5e439c9c40572 gencode/docs/event_pointset.html @@ -8,7 +8,7 @@ a82821e72af6d0ee35800e6262eb9bb05256309b98aed2dad1a368fd2d6882bb gencode/docs/i 741b880216be3743f6747800a042f2dbd89f3b0344c6b0a965f4bc010f03a930 gencode/docs/schema_doc.css 878ea88206c974f40643c3cc430875f9c4e8c5e3fd6bcd6358bd3eb6d48699a9 gencode/docs/schema_doc.min.js 7ed934930aee763e0beebc349725ba3909115e8d346bb762f28bcbe745bb163a gencode/docs/schema_extras.js -cbbff5d74adb59cd83bfb37813b008a9970c239b2fb1084dc8a020b8856eac01 gencode/docs/state.html +43019239edfdad71647c40d254b9b3aec68f3deb0c808ba636530c1cf9985fe0 gencode/docs/state.html d39d7fe37a41c74a40080af7b0a429d201ab1fdff7444428c4b98eb7b38c332b gencode/java/udmi/schema/Asset.java 0edc9b905bbe4ad0ee5ffdb7fb5d65a00ad171b2dad299ef128e59c0bd9dddc0 gencode/java/udmi/schema/AuditEvent.java 0825a5cec83003bb0a6488c4ed7010a04ae0d3848ef36fe01bb4e6718ba7b96d gencode/java/udmi/schema/Aux.java From 4ec47feede18b1bc56913cf719eb9ba5a5db31d4 Mon Sep 17 00:00:00 2001 From: John Randolph Date: Thu, 28 Jul 2022 11:35:42 -0400 Subject: [PATCH 55/57] Make the regex more accurate (a-f) and specific (lowercase) --- schema/config_blobset_blob.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schema/config_blobset_blob.json b/schema/config_blobset_blob.json index bc8801d315..ed5b2d284b 100644 --- a/schema/config_blobset_blob.json +++ b/schema/config_blobset_blob.json @@ -20,7 +20,7 @@ }, "sha256": { "type": "string", - "pattern": "^[0-9A-Za-z]{64}$" + "pattern": "^[0-9a-f]{64}$" } }, "oneOf": [ From 02a9fad69c2be9454ca4431c11aec2cf0b4dbb79 Mon Sep 17 00:00:00 2001 From: John Randolph Date: Thu, 28 Jul 2022 11:50:50 -0400 Subject: [PATCH 56/57] Put this back the way it was for this submit --- bin/check_version | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/bin/check_version b/bin/check_version index a076adb425..f574ceecfe 100755 --- a/bin/check_version +++ b/bin/check_version @@ -15,14 +15,10 @@ if [[ -z $origin ]]; then origin=faucetsdn fi -git fetch $origin --tags # 2>/dev/null +git fetch $origin --tags 2>/dev/null -if [[ $? -ne 0 ]]; then - current_ver="unknown" -else - upstream_ver=`git describe $origin/$branch --abbrev=0 || echo unknown` - current_ver=`git describe HEAD --abbrev=0 || echo unknown` -fi +upstream_ver=`git describe $origin/$branch --abbrev=0 || echo unknown` +current_ver=`git describe HEAD --abbrev=0 || echo unknown` if [[ $current_ver != $upstream_ver ]]; then echo From 5cdfa1e1dd219bd5a81286722fbcd1aba472f92f Mon Sep 17 00:00:00 2001 From: John Randolph Date: Thu, 28 Jul 2022 12:07:55 -0400 Subject: [PATCH 57/57] Updates because of a-z to a-f change --- .gencode_hash.txt | 2 +- gencode/docs/config.html | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gencode_hash.txt b/.gencode_hash.txt index 3c14421adc..3866342eea 100644 --- a/.gencode_hash.txt +++ b/.gencode_hash.txt @@ -1,4 +1,4 @@ -8c0f20d1da08192edceb6c0c3d3b6de85a2cc5abd5d8188fc56000605871de7b gencode/docs/config.html +3cd595a2ef2e148e6b1a4e7e12a52c4da8b6ecf1222826ca5656ace77be82d05 gencode/docs/config.html e2944b13db5ff06be9caea51d03bca48f2cb093a8bb583dca14051255d34ea6b gencode/docs/envelope.html 04b55d0e207495a1c471423a5e89347342241bbd3c845de3e8355209692eb70a gencode/docs/event_discovery.html 8133e380e40f27c56accbffc665b2eeb56ec84a4da3b52ba7aa5e439c9c40572 gencode/docs/event_pointset.html diff --git a/gencode/docs/config.html b/gencode/docs/config.html index 04d570e780..566a8318d0 100644 --- a/gencode/docs/config.html +++ b/gencode/docs/config.html @@ -1469,6 +1469,7 @@

class="collapse property-definition-div" aria-labelledby="headingblobset_blobs_pattern1_base64" data-parent="#accordionblobset_blobs_pattern1_base64">
+
root /> sha256
Type: string
-Must match regular expression: ^[0-9A-Za-z]{64}$ +Must match regular expression: ^[0-9a-f]{64}$