From a713dc0c145f0b99a6efda49dfcb6e86f027ab17 Mon Sep 17 00:00:00 2001 From: John Randolph Date: Thu, 31 Mar 2022 14:51:15 -0400 Subject: [PATCH 01/16] Add site_defaults so that sample_rate_sec, sample_limit_sec, min_loglevel can have default settings for the entire set. --- gencode/java/udmi/schema/SiteDefaults.java | 69 +++++++++++ schema/site_defaults.json | 26 +++++ .../daq/mqtt/registrar/LocalDevice.java | 107 ++++++++++++------ .../google/daq/mqtt/registrar/Registrar.java | 43 +++++-- 4 files changed, 203 insertions(+), 42 deletions(-) create mode 100644 gencode/java/udmi/schema/SiteDefaults.java create mode 100644 schema/site_defaults.json diff --git a/gencode/java/udmi/schema/SiteDefaults.java b/gencode/java/udmi/schema/SiteDefaults.java new file mode 100644 index 0000000000..b9184b7a2a --- /dev/null +++ b/gencode/java/udmi/schema/SiteDefaults.java @@ -0,0 +1,69 @@ + +package udmi.schema; + +import javax.annotation.processing.Generated; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + + +/** + * Site Defaults + *

+ * + * + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder({ + "min_loglevel", + "sample_limit_sec", + "sample_rate_sec" +}) +@Generated("jsonschema2pojo") +public class SiteDefaults { + + /** + * The minimum loglevel for reporting log messages below which log entries should not be sent. Default to 300. + * + */ + @JsonProperty("min_loglevel") + @JsonPropertyDescription("The minimum loglevel for reporting log messages below which log entries should not be sent. Default to 300.") + public Integer min_loglevel; + /** + * Minimum time between sample updates for the device (including complete and COV updates). Updates more frequent than this should be coalesced into one update. + * + */ + @JsonProperty("sample_limit_sec") + @JsonPropertyDescription("Minimum time between sample updates for the device (including complete and COV updates). Updates more frequent than this should be coalesced into one update.") + public Integer sample_limit_sec; + /** + * Maximum time between samples for the device to send out a complete update. It can send out updates more frequently than this. Default to 600. + * + */ + @JsonProperty("sample_rate_sec") + @JsonPropertyDescription("Maximum time between samples for the device to send out a complete update. It can send out updates more frequently than this. Default to 600.") + public Integer sample_rate_sec; + + @Override + public int hashCode() { + int result = 1; + result = ((result* 31)+((this.sample_rate_sec == null)? 0 :this.sample_rate_sec.hashCode())); + result = ((result* 31)+((this.min_loglevel == null)? 0 :this.min_loglevel.hashCode())); + result = ((result* 31)+((this.sample_limit_sec == null)? 0 :this.sample_limit_sec.hashCode())); + return result; + } + + @Override + public boolean equals(Object other) { + if (other == this) { + return true; + } + if ((other instanceof SiteDefaults) == false) { + return false; + } + SiteDefaults rhs = ((SiteDefaults) other); + return ((((this.sample_rate_sec == rhs.sample_rate_sec)||((this.sample_rate_sec!= null)&&this.sample_rate_sec.equals(rhs.sample_rate_sec)))&&((this.min_loglevel == rhs.min_loglevel)||((this.min_loglevel!= null)&&this.min_loglevel.equals(rhs.min_loglevel))))&&((this.sample_limit_sec == rhs.sample_limit_sec)||((this.sample_limit_sec!= null)&&this.sample_limit_sec.equals(rhs.sample_limit_sec)))); + } + +} diff --git a/schema/site_defaults.json b/schema/site_defaults.json new file mode 100644 index 0000000000..558887ff15 --- /dev/null +++ b/schema/site_defaults.json @@ -0,0 +1,26 @@ +{ + "title": "Site Defaults", + "type": "object", + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "min_loglevel": { + "description": "The minimum loglevel for reporting log messages below which log entries should not be sent. Default to 300.", + "type": "integer", + "minimum": 100, + "maximum": 800 + }, + "sample_limit_sec": { + "description": "Minimum time between sample updates for the device (including complete and COV updates). Updates more frequent than this should be coalesced into one update.", + "type": "integer", + "minimum": 0, + "maximum": 86400 + }, + "sample_rate_sec": { + "description": "Maximum time between samples for the device to send out a complete update. It can send out updates more frequently than this. Default to 600.", + "type": "integer", + "minimum": 1, + "maximum": 86400 + } + } +} diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java b/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java index 022dcbc9d5..466d42b2b6 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java @@ -64,20 +64,12 @@ import udmi.schema.PointPointsetConfig; import udmi.schema.PointPointsetMetadata; import udmi.schema.PointsetConfig; +import udmi.schema.SiteDefaults; class LocalDevice { - public static final String INVALID_METADATA_HASH = "INVALID"; - public static final String EXCEPTION_VALIDATING = "Validating"; - public static final String EXCEPTION_LOADING = "Loading"; - public static final String EXCEPTION_READING = "Reading"; - public static final String EXCEPTION_WRITING = "Writing"; - public static final String EXCEPTION_FILES = "Files"; - public static final String EXCEPTION_REGISTERING = "Registering"; - public static final String EXCEPTION_CREDENTIALS = "Credential"; - public static final String EXCEPTION_ENVELOPE = "Envelope"; - public static final String EXCEPTION_SAMPLES = "Samples"; private static final PrettyPrinter PROPER_PRETTY_PRINTER_POLICY = new ProperPrettyPrinterPolicy(); + private static final ObjectMapper OBJECT_MAPPER_RAW = new ObjectMapper() .enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS) @@ -86,11 +78,13 @@ class LocalDevice { .disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS) .setDateFormat(new ISO8601DateFormat()) .setSerializationInclusion(Include.NON_NULL); + private static final ObjectMapper OBJECT_MAPPER = OBJECT_MAPPER_RAW .copy() .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) .enable(SerializationFeature.INDENT_OUTPUT); + private static final String RSA_AUTH_TYPE = "RS256"; private static final String RSA_CERT_TYPE = "RS256_X509"; private static final String RSA_KEY_FORMAT = "RSA_PEM"; @@ -101,6 +95,7 @@ class LocalDevice { private static final String RSA_CERT_PEM = "rsa_cert.pem"; private static final String RSA_PRIVATE_PEM = "rsa_private.pem"; private static final String RSA_PRIVATE_PKCS8 = "rsa_private.pkcs8"; + private static final String ES_AUTH_TYPE = "ES256"; private static final String ES_CERT_TYPE = "ES256_X509"; private static final String ES_KEY_FORMAT = "ES256_PEM"; @@ -111,17 +106,14 @@ class LocalDevice { private static final String ES_CERT_PEM = "ec_cert.pem"; private static final String ES_PRIVATE_PEM = "ec_private.pem"; private static final String ES_PRIVATE_PKCS8 = "ec_private.pkcs8"; - protected static final Map PRIVATE_PKCS8_MAP = - ImmutableMap.of( - RSA_AUTH_TYPE, RSA_PRIVATE_PKCS8, - RSA_CERT_TYPE, RSA_PRIVATE_PKCS8, - ES_AUTH_TYPE, ES_PRIVATE_PKCS8, - ES_CERT_TYPE, ES_PRIVATE_PKCS8); + private static final String SAMPLES_DIR = "samples"; private static final String AUX_DIR = "aux"; private static final String OUT_DIR = "out"; private static final String EXCEPTION_LOG_FILE = "exceptions.txt"; + private static final Set DEVICE_FILES = ImmutableSet.of(METADATA_JSON); + private static final Set RSA_PRIVATE_KEY_FILES = ImmutableSet.of(RSA_PRIVATE_PEM, RSA_PRIVATE_PKCS8); private static final Set ES_PRIVATE_KEY_FILES = @@ -132,6 +124,14 @@ class LocalDevice { RSA_CERT_TYPE, RSA_PRIVATE_KEY_FILES, ES_AUTH_TYPE, ES_PRIVATE_KEY_FILES, ES_CERT_TYPE, ES_PRIVATE_KEY_FILES); + + protected static final Map PRIVATE_PKCS8_MAP = + ImmutableMap.of( + RSA_AUTH_TYPE, RSA_PRIVATE_PKCS8, + RSA_CERT_TYPE, RSA_PRIVATE_PKCS8, + ES_AUTH_TYPE, ES_PRIVATE_PKCS8, + ES_CERT_TYPE, ES_PRIVATE_PKCS8); + private static final Map PUBLIC_KEY_FILE_MAP = ImmutableMap.of( RSA_AUTH_TYPE, RSA_PUBLIC_PEM, @@ -151,8 +151,10 @@ class LocalDevice { SAMPLES_DIR, AUX_DIR, OUT_DIR); + private static final Set OUT_FILES = ImmutableSet.of(GENERATED_CONFIG_JSON, DEVICE_ERRORS_JSON, NORMALIZED_JSON); + private static final Set ALL_KEY_FILES = ImmutableSet.of( RSA_PUBLIC_PEM, @@ -168,9 +170,23 @@ class LocalDevice { RSA_CERT_TYPE, RSA_CERT_FORMAT, ES_AUTH_TYPE, ES_KEY_FORMAT, ES_CERT_TYPE, ES_CERT_FILE); + private static final String ERROR_FORMAT_INDENT = " "; private static final int MAX_METADATA_LENGTH = 32767; - public static final String UDMI_VERSION = "1.3.14"; + private static final String UDMI_VERSION = "1.3.14"; + + public static final String INVALID_METADATA_HASH = "INVALID"; + + public static final String EXCEPTION_VALIDATING = "Validating"; + public static final String EXCEPTION_LOADING = "Loading"; + public static final String EXCEPTION_READING = "Reading"; + public static final String EXCEPTION_WRITING = "Writing"; + public static final String EXCEPTION_FILES = "Files"; + public static final String EXCEPTION_REGISTERING = "Registering"; + public static final String EXCEPTION_CREDENTIALS = "Credential"; + public static final String EXCEPTION_ENVELOPE = "Envelope"; + public static final String EXCEPTION_SAMPLES = "Samples"; + private final String deviceId; private final Map schemas; private final File siteDir; @@ -180,6 +196,7 @@ class LocalDevice { private final ExceptionMap exceptionMap; private final String generation; private final List deviceCredentials = new ArrayList<>(); + private final SiteDefaults siteDefaults; private String deviceNumId; @@ -187,12 +204,13 @@ class LocalDevice { LocalDevice( File siteDir, File devicesDir, String deviceId, Map schemas, - String generation) { + String generation, SiteDefaults siteDefaults) { try { this.deviceId = deviceId; this.schemas = schemas; this.generation = generation; this.siteDir = siteDir; + this.siteDefaults = siteDefaults; exceptionMap = new ExceptionMap("Exceptions for " + deviceId); deviceDir = new File(devicesDir, deviceId); outDir = new File(deviceDir, OUT_DIR); @@ -203,8 +221,10 @@ class LocalDevice { } } - static boolean deviceExists(File devicesDir, String deviceName) { - return new File(new File(devicesDir, deviceName), METADATA_JSON).isFile(); + LocalDevice( + File siteDir, File devicesDir, String deviceId, Map schemas, + String generation) { + this(siteDir, devicesDir, deviceId, schemas, generation, null); } private void prepareOutDir() { @@ -215,6 +235,10 @@ private void prepareOutDir() { exceptionLog.delete(); } + static boolean deviceExists(File devicesDir, String deviceName) { + return new File(new File(devicesDir, deviceName), METADATA_JSON).isFile(); + } + public void validateExpected() { Path relativized = siteDir.toPath().relativize(deviceDir.toPath()); ExceptionMap exceptionMap = new ExceptionMap(relativized.toString()); @@ -244,28 +268,49 @@ public void validateExpected() { exceptionMap.throwIfNotEmpty(); } - private Metadata readMetadata() { + private Metadata readMetadataBase() { File metadataFile = new File(deviceDir, METADATA_JSON); try (InputStream targetStream = new FileInputStream(metadataFile)) { schemas.get(METADATA_JSON).validate(OBJECT_MAPPER.readTree(targetStream)); - } catch (ProcessingException | ValidationException metadataException) { - exceptionMap.put(EXCEPTION_VALIDATING, metadataException); + } catch (ProcessingException | ValidationException metadata_exception) { + exceptionMap.put(EXCEPTION_VALIDATING, metadata_exception); } catch (IOException ioException) { exceptionMap.put(EXCEPTION_LOADING, ioException); } try { return OBJECT_MAPPER.readValue(metadataFile, Metadata.class); - } catch (Exception mappingException) { - exceptionMap.put(EXCEPTION_READING, mappingException); + } catch (Exception mapping_exception) { + exceptionMap.put(EXCEPTION_READING, mapping_exception); } return null; } + private Metadata readMetadata() { + final Metadata metadata = readMetadataBase(); + // Copy values from siteDefaults into the metadata + + if (siteDefaults != null) { + // Fields in siteDefaults that go into PointPointsetMetadata + for (String pointName : metadata.pointset.points.keySet()) { + PointPointsetMetadata ppm = metadata.pointset.points.get(pointName); + if (ppm.min_loglevel == null) + ppm.min_loglevel = siteDefaults.min_loglevel; + if (ppm.sample_limit_sec == null) + ppm.sample_limit_sec = siteDefaults.sample_limit_sec; + if (ppm.sample_rate_sec == null) + ppm.sample_rate_sec = siteDefaults.sample_rate_sec; + metadata.pointset.points.put(pointName, ppm); + } + } + + return metadata; + } + private Metadata readNormalized() { try { File metadataFile = new File(outDir, NORMALIZED_JSON); return OBJECT_MAPPER.readValue(metadataFile, Metadata.class); - } catch (Exception mappingException) { + } catch (Exception mapping_exception) { return new Metadata(); } } @@ -510,11 +555,11 @@ private PointsetConfig getDevicePointsetConfig() { metadata.pointset.points.forEach( (metadataKey, value) -> pointsetConfig.points.computeIfAbsent( - metadataKey, configKey -> configFromMetadata(value))); + metadataKey, configKey -> ConfigFromMetadata(value))); return pointsetConfig; } - PointPointsetConfig configFromMetadata(PointPointsetMetadata metadata) { + PointPointsetConfig ConfigFromMetadata(PointPointsetMetadata metadata) { PointPointsetConfig pointConfig = new PointPointsetConfig(); pointConfig.ref = metadata.ref; if (Boolean.TRUE.equals(metadata.writable)) { @@ -550,8 +595,7 @@ public void validateEnvelope(String registryId, String siteName) { envelope.projectId = fakeProjectId(); envelope.deviceNumId = makeNumId(envelope); String envelopeJson = OBJECT_MAPPER.writeValueAsString(envelope); - ProcessingReport processingReport = schemas.get(ENVELOPE_JSON) - .validate(OBJECT_MAPPER.readTree(envelopeJson)); + ProcessingReport processingReport = schemas.get(ENVELOPE_JSON).validate(OBJECT_MAPPER.readTree(envelopeJson)); if (!processingReport.isSuccess()) { processingReport.forEach(action -> { throw new RuntimeException("against schema", action.asException()); @@ -690,7 +734,7 @@ public void captureError(String exceptionType, Exception exception) { File exceptionLog = new File(outDir, EXCEPTION_LOG_FILE); try { try (FileWriter fileWriter = new FileWriter(exceptionLog, true); - PrintWriter printWriter = new PrintWriter(fileWriter)) { + PrintWriter printWriter = new PrintWriter(fileWriter)) { printWriter.println(exceptionType); exception.printStackTrace(printWriter); } @@ -742,7 +786,6 @@ public Metadata getMetadata() { } private static class ProperPrettyPrinterPolicy extends DefaultPrettyPrinter { - @Override public void writeObjectFieldValueSeparator(JsonGenerator jg) throws IOException { jg.writeRaw(": "); diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java b/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java index 4cb1f743c1..8757f8c5f6 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java @@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.databind.util.ISO8601DateFormat; +import com.github.fge.jsonschema.core.exceptions.ProcessingException; import com.github.fge.jsonschema.core.load.configuration.LoadingConfiguration; import com.github.fge.jsonschema.core.load.download.URIDownloader; import com.github.fge.jsonschema.main.JsonSchema; @@ -17,16 +18,10 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; -import com.google.daq.mqtt.util.CloudDeviceSettings; -import com.google.daq.mqtt.util.CloudIotManager; -import com.google.daq.mqtt.util.ConfigUtil; -import com.google.daq.mqtt.util.ExceptionMap; +import com.google.daq.mqtt.util.*; import com.google.daq.mqtt.util.ExceptionMap.ErrorTree; -import com.google.daq.mqtt.util.PubSubPusher; -import java.io.File; -import java.io.FileInputStream; -import java.io.FilenameFilter; -import java.io.InputStream; + +import java.io.*; import java.math.BigInteger; import java.net.URI; import java.time.Duration; @@ -49,6 +44,8 @@ import java.util.stream.Collectors; import udmi.schema.Config; import udmi.schema.Envelope.SubFolder; +import udmi.schema.Metadata; +import udmi.schema.SiteDefaults; public class Registrar { @@ -73,6 +70,7 @@ public class Registrar { private static final String SCHEMA_SUFFIX = ".json"; private static final String REGISTRATION_SUMMARY_JSON = "registration_summary.json"; private static final String SCHEMA_NAME = "UDMI"; + private static final String SITE_DEFAULTS_JSON = "site_defaults.json"; private static final String SWARM_SUBFOLDER = "swarm"; private static final long PROCESSING_TIMEOUT_MIN = 60; private final Map schemas = new HashMap<>(); @@ -89,6 +87,7 @@ public class Registrar { private boolean updateCloudIoT; private Duration idleLimit; private Set cloudDevices; + private SiteDefaults siteDefaults; public static void main(String[] args) { ArrayList argList = new ArrayList<>(List.of(args)); @@ -100,6 +99,8 @@ public static void main(String[] args) { registrar.setToolRoot(null); } + registrar.loadSiteDefaults(); + if (processAllDevices) { registrar.processDevices(); } else { @@ -597,7 +598,7 @@ private Map loadDevices(File siteDir, File devicesDir, LocalDevice localDevice = localDevices.computeIfAbsent( deviceName, - keyName -> new LocalDevice(siteDir, devicesDir, deviceName, schemas, generation)); + keyName -> new LocalDevice(siteDir, devicesDir, deviceName, schemas, generation, siteDefaults)); try { localDevice.loadCredentials(); } catch (Exception e) { @@ -655,6 +656,28 @@ private void loadSchema(String key) { } } + private void loadSiteDefaults() { + if (!schemas.containsKey("site_defaults.json")) + return; + + File siteDefaultsFile = new File(siteDir, SITE_DEFAULTS_JSON); + try (InputStream targetStream = new FileInputStream(siteDefaultsFile)) { + schemas.get(SITE_DEFAULTS_JSON).validate(OBJECT_MAPPER.readTree(targetStream)); + } catch (ProcessingException | ValidationException e) { + throw new RuntimeException("While validating " + SITE_DEFAULTS_JSON, e); + } catch (IOException e) { + throw new RuntimeException("While validating " + SITE_DEFAULTS_JSON, e); + } + + this.siteDefaults = null; + + try { + this.siteDefaults = OBJECT_MAPPER.readValue(siteDefaultsFile, SiteDefaults.class); + } catch (Exception e) { + throw new RuntimeException("While loading " + SITE_DEFAULTS_JSON, e); + } + } + class RelativeDownloader implements URIDownloader { @Override From 87a67b7a50729a0cec98fa93fa1ae4212ae2e6d3 Mon Sep 17 00:00:00 2001 From: John Randolph Date: Wed, 6 Apr 2022 13:12:24 -0400 Subject: [PATCH 02/16] Load site_defaults.json into site-wide Metadata. Merge values from site_defaults.json into device Metadata if the values are not present. --- .../daq/mqtt/registrar/LocalDevice.java | 58 +++++++++++++------ .../google/daq/mqtt/registrar/Registrar.java | 11 ++-- 2 files changed, 45 insertions(+), 24 deletions(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java b/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java index 466d42b2b6..c50833cac2 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java @@ -42,16 +42,8 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.Map.Entry; -import java.util.Objects; -import java.util.Set; -import java.util.SortedSet; -import java.util.TreeSet; import java.util.regex.Pattern; import org.apache.commons.io.IOUtils; import udmi.schema.Config; @@ -196,7 +188,7 @@ class LocalDevice { private final ExceptionMap exceptionMap; private final String generation; private final List deviceCredentials = new ArrayList<>(); - private final SiteDefaults siteDefaults; + private final TreeMap siteDefaults; private String deviceNumId; @@ -204,13 +196,17 @@ class LocalDevice { LocalDevice( File siteDir, File devicesDir, String deviceId, Map schemas, - String generation, SiteDefaults siteDefaults) { + String generation, Metadata siteDefaults) { try { this.deviceId = deviceId; this.schemas = schemas; this.generation = generation; this.siteDir = siteDir; - this.siteDefaults = siteDefaults; + if (siteDefaults != null) { + this.siteDefaults = OBJECT_MAPPER.convertValue(siteDefaults, TreeMap.class); + } else { + this.siteDefaults = new TreeMap(); + } exceptionMap = new ExceptionMap("Exceptions for " + deviceId); deviceDir = new File(devicesDir, deviceId); outDir = new File(deviceDir, OUT_DIR); @@ -268,7 +264,26 @@ public void validateExpected() { exceptionMap.throwIfNotEmpty(); } - private Metadata readMetadataBase() { + private List listMerge(List list1, List list2) { + list2.removeAll(list1); + list1.addAll(list2); + return list1; + } + private void deepMerge(Map map1, Map map2) { + for(String key : map2.keySet()) { + Object value2 = map2.get(key); + if (map1.containsKey(key)) { + Object value1 = map1.get(key); + if (value1 instanceof Map && value2 instanceof Map) + deepMerge((Map) value1, (Map) value2); + else if (value1 instanceof List && value2 instanceof List) + map1.put(key, listMerge((List) value1, (List) value2)); + else map1.put(key, value2); + } else map1.put(key, value2); + } + } + + private Metadata readMetadataBase(Map siteDefaults) { File metadataFile = new File(deviceDir, METADATA_JSON); try (InputStream targetStream = new FileInputStream(metadataFile)) { schemas.get(METADATA_JSON).validate(OBJECT_MAPPER.readTree(targetStream)); @@ -278,7 +293,13 @@ private Metadata readMetadataBase() { exceptionMap.put(EXCEPTION_LOADING, ioException); } try { - return OBJECT_MAPPER.readValue(metadataFile, Metadata.class); + if (siteDefaults == null) { + return OBJECT_MAPPER.readValue(metadataFile, Metadata.class); + } else { + final Map metadata_base = OBJECT_MAPPER.readValue(metadataFile, TreeMap.class); + deepMerge(metadata_base, siteDefaults); + return OBJECT_MAPPER.convertValue(metadata_base, Metadata.class); + } } catch (Exception mapping_exception) { exceptionMap.put(EXCEPTION_READING, mapping_exception); } @@ -286,9 +307,10 @@ private Metadata readMetadataBase() { } private Metadata readMetadata() { - final Metadata metadata = readMetadataBase(); - // Copy values from siteDefaults into the metadata + return readMetadataBase(siteDefaults); + + /* if (siteDefaults != null) { // Fields in siteDefaults that go into PointPointsetMetadata for (String pointName : metadata.pointset.points.keySet()) { @@ -301,9 +323,7 @@ private Metadata readMetadata() { ppm.sample_rate_sec = siteDefaults.sample_rate_sec; metadata.pointset.points.put(pointName, ppm); } - } - - return metadata; + }*/ } private Metadata readNormalized() { diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java b/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java index 8757f8c5f6..8172669017 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java @@ -63,7 +63,8 @@ public class Registrar { .enable(SerializationFeature.INDENT_OUTPUT) .disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS) .setDateFormat(new ISO8601DateFormat()) - .setSerializationInclusion(Include.NON_NULL); + .setSerializationInclusion(Include.NON_NULL) + .setDefaultMergeable(Boolean.TRUE); private static final String UDMI_VERSION_KEY = "UDMI_VERSION"; private static final String VERSION_KEY = "Version"; private static final String VERSION_MAIN_KEY = "main"; @@ -87,7 +88,7 @@ public class Registrar { private boolean updateCloudIoT; private Duration idleLimit; private Set cloudDevices; - private SiteDefaults siteDefaults; + private Metadata siteDefaults; public static void main(String[] args) { ArrayList argList = new ArrayList<>(List.of(args)); @@ -657,12 +658,12 @@ private void loadSchema(String key) { } private void loadSiteDefaults() { - if (!schemas.containsKey("site_defaults.json")) + if (!schemas.containsKey(METADATA_JSON)) return; File siteDefaultsFile = new File(siteDir, SITE_DEFAULTS_JSON); try (InputStream targetStream = new FileInputStream(siteDefaultsFile)) { - schemas.get(SITE_DEFAULTS_JSON).validate(OBJECT_MAPPER.readTree(targetStream)); + schemas.get(METADATA_JSON).validate(OBJECT_MAPPER.readTree(targetStream)); } catch (ProcessingException | ValidationException e) { throw new RuntimeException("While validating " + SITE_DEFAULTS_JSON, e); } catch (IOException e) { @@ -672,7 +673,7 @@ private void loadSiteDefaults() { this.siteDefaults = null; try { - this.siteDefaults = OBJECT_MAPPER.readValue(siteDefaultsFile, SiteDefaults.class); + this.siteDefaults = OBJECT_MAPPER.readValue(siteDefaultsFile, Metadata.class); } catch (Exception e) { throw new RuntimeException("While loading " + SITE_DEFAULTS_JSON, e); } From cfc48f01f69a0fee39470058749b89352c2aac0b Mon Sep 17 00:00:00 2001 From: John Randolph Date: Wed, 6 Apr 2022 13:19:42 -0400 Subject: [PATCH 03/16] Set null instead of making a map instance. --- .../main/java/com/google/daq/mqtt/registrar/LocalDevice.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java b/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java index c50833cac2..c31418b51a 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java @@ -205,7 +205,7 @@ class LocalDevice { if (siteDefaults != null) { this.siteDefaults = OBJECT_MAPPER.convertValue(siteDefaults, TreeMap.class); } else { - this.siteDefaults = new TreeMap(); + this.siteDefaults = null; } exceptionMap = new ExceptionMap("Exceptions for " + deviceId); deviceDir = new File(devicesDir, deviceId); From 93545a6c7c047057612057e0d09c0a57b38b93b0 Mon Sep 17 00:00:00 2001 From: John Randolph Date: Wed, 6 Apr 2022 13:24:44 -0400 Subject: [PATCH 04/16] Remove site_defaults schema which is no longer needed, because site_defaults.json is loaded with the metadata schema. --- gencode/java/udmi/schema/SiteDefaults.java | 69 ------------------- schema/site_defaults.json | 26 ------- .../daq/mqtt/registrar/LocalDevice.java | 1 - .../google/daq/mqtt/registrar/Registrar.java | 1 - 4 files changed, 97 deletions(-) delete mode 100644 gencode/java/udmi/schema/SiteDefaults.java delete mode 100644 schema/site_defaults.json diff --git a/gencode/java/udmi/schema/SiteDefaults.java b/gencode/java/udmi/schema/SiteDefaults.java deleted file mode 100644 index b9184b7a2a..0000000000 --- a/gencode/java/udmi/schema/SiteDefaults.java +++ /dev/null @@ -1,69 +0,0 @@ - -package udmi.schema; - -import javax.annotation.processing.Generated; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonPropertyDescription; -import com.fasterxml.jackson.annotation.JsonPropertyOrder; - - -/** - * Site Defaults - *

- * - * - */ -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonPropertyOrder({ - "min_loglevel", - "sample_limit_sec", - "sample_rate_sec" -}) -@Generated("jsonschema2pojo") -public class SiteDefaults { - - /** - * The minimum loglevel for reporting log messages below which log entries should not be sent. Default to 300. - * - */ - @JsonProperty("min_loglevel") - @JsonPropertyDescription("The minimum loglevel for reporting log messages below which log entries should not be sent. Default to 300.") - public Integer min_loglevel; - /** - * Minimum time between sample updates for the device (including complete and COV updates). Updates more frequent than this should be coalesced into one update. - * - */ - @JsonProperty("sample_limit_sec") - @JsonPropertyDescription("Minimum time between sample updates for the device (including complete and COV updates). Updates more frequent than this should be coalesced into one update.") - public Integer sample_limit_sec; - /** - * Maximum time between samples for the device to send out a complete update. It can send out updates more frequently than this. Default to 600. - * - */ - @JsonProperty("sample_rate_sec") - @JsonPropertyDescription("Maximum time between samples for the device to send out a complete update. It can send out updates more frequently than this. Default to 600.") - public Integer sample_rate_sec; - - @Override - public int hashCode() { - int result = 1; - result = ((result* 31)+((this.sample_rate_sec == null)? 0 :this.sample_rate_sec.hashCode())); - result = ((result* 31)+((this.min_loglevel == null)? 0 :this.min_loglevel.hashCode())); - result = ((result* 31)+((this.sample_limit_sec == null)? 0 :this.sample_limit_sec.hashCode())); - return result; - } - - @Override - public boolean equals(Object other) { - if (other == this) { - return true; - } - if ((other instanceof SiteDefaults) == false) { - return false; - } - SiteDefaults rhs = ((SiteDefaults) other); - return ((((this.sample_rate_sec == rhs.sample_rate_sec)||((this.sample_rate_sec!= null)&&this.sample_rate_sec.equals(rhs.sample_rate_sec)))&&((this.min_loglevel == rhs.min_loglevel)||((this.min_loglevel!= null)&&this.min_loglevel.equals(rhs.min_loglevel))))&&((this.sample_limit_sec == rhs.sample_limit_sec)||((this.sample_limit_sec!= null)&&this.sample_limit_sec.equals(rhs.sample_limit_sec)))); - } - -} diff --git a/schema/site_defaults.json b/schema/site_defaults.json deleted file mode 100644 index 558887ff15..0000000000 --- a/schema/site_defaults.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "title": "Site Defaults", - "type": "object", - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": false, - "properties": { - "min_loglevel": { - "description": "The minimum loglevel for reporting log messages below which log entries should not be sent. Default to 300.", - "type": "integer", - "minimum": 100, - "maximum": 800 - }, - "sample_limit_sec": { - "description": "Minimum time between sample updates for the device (including complete and COV updates). Updates more frequent than this should be coalesced into one update.", - "type": "integer", - "minimum": 0, - "maximum": 86400 - }, - "sample_rate_sec": { - "description": "Maximum time between samples for the device to send out a complete update. It can send out updates more frequently than this. Default to 600.", - "type": "integer", - "minimum": 1, - "maximum": 86400 - } - } -} diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java b/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java index c31418b51a..1db96de9c7 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java @@ -56,7 +56,6 @@ import udmi.schema.PointPointsetConfig; import udmi.schema.PointPointsetMetadata; import udmi.schema.PointsetConfig; -import udmi.schema.SiteDefaults; class LocalDevice { diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java b/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java index 8172669017..962a55a62d 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java @@ -45,7 +45,6 @@ import udmi.schema.Config; import udmi.schema.Envelope.SubFolder; import udmi.schema.Metadata; -import udmi.schema.SiteDefaults; public class Registrar { From bed4cffabb355a43dc22cb3eb68ef04d509096a0 Mon Sep 17 00:00:00 2001 From: John Randolph Date: Wed, 6 Apr 2022 13:27:51 -0400 Subject: [PATCH 05/16] Restore readMetadata() and remove intermediate readMetadataBase(), the latter of which existed to post-process readMetadata() output in an earlier version. --- .../daq/mqtt/registrar/LocalDevice.java | 24 ++----------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java b/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java index 1db96de9c7..7b27cd34e3 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java @@ -282,7 +282,7 @@ else if (value1 instanceof List && value2 instanceof List) } } - private Metadata readMetadataBase(Map siteDefaults) { + private Metadata readMetadata() { File metadataFile = new File(deviceDir, METADATA_JSON); try (InputStream targetStream = new FileInputStream(metadataFile)) { schemas.get(METADATA_JSON).validate(OBJECT_MAPPER.readTree(targetStream)); @@ -304,27 +304,7 @@ private Metadata readMetadataBase(Map siteDefaults) { } return null; } - - private Metadata readMetadata() { - return readMetadataBase(siteDefaults); - - - /* - if (siteDefaults != null) { - // Fields in siteDefaults that go into PointPointsetMetadata - for (String pointName : metadata.pointset.points.keySet()) { - PointPointsetMetadata ppm = metadata.pointset.points.get(pointName); - if (ppm.min_loglevel == null) - ppm.min_loglevel = siteDefaults.min_loglevel; - if (ppm.sample_limit_sec == null) - ppm.sample_limit_sec = siteDefaults.sample_limit_sec; - if (ppm.sample_rate_sec == null) - ppm.sample_rate_sec = siteDefaults.sample_rate_sec; - metadata.pointset.points.put(pointName, ppm); - } - }*/ - } - + private Metadata readNormalized() { try { File metadataFile = new File(outDir, NORMALIZED_JSON); From 4fa3f81b3bbeb3873930d65b67b1d80e2387f961 Mon Sep 17 00:00:00 2001 From: John Randolph Date: Wed, 6 Apr 2022 13:31:05 -0400 Subject: [PATCH 06/16] Cleanup in Registrar loadSiteDefaults(). --- .../main/java/com/google/daq/mqtt/registrar/LocalDevice.java | 2 +- .../main/java/com/google/daq/mqtt/registrar/Registrar.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java b/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java index 7b27cd34e3..81d1f47204 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java @@ -304,7 +304,7 @@ private Metadata readMetadata() { } return null; } - + private Metadata readNormalized() { try { File metadataFile = new File(outDir, NORMALIZED_JSON); diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java b/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java index 962a55a62d..3e94e316f1 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java @@ -657,6 +657,8 @@ private void loadSchema(String key) { } private void loadSiteDefaults() { + this.siteDefaults = null; + if (!schemas.containsKey(METADATA_JSON)) return; @@ -669,8 +671,6 @@ private void loadSiteDefaults() { throw new RuntimeException("While validating " + SITE_DEFAULTS_JSON, e); } - this.siteDefaults = null; - try { this.siteDefaults = OBJECT_MAPPER.readValue(siteDefaultsFile, Metadata.class); } catch (Exception e) { From aeb0ac927494f1a94744091567c8dffe2c3eaff6 Mon Sep 17 00:00:00 2001 From: John Randolph Date: Wed, 6 Apr 2022 13:32:56 -0400 Subject: [PATCH 07/16] Remove setDefaultMergeable() property set, an earlier attempt to produce default configs. --- .../src/main/java/com/google/daq/mqtt/registrar/Registrar.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java b/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java index 3e94e316f1..e665e2b010 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java @@ -62,8 +62,7 @@ public class Registrar { .enable(SerializationFeature.INDENT_OUTPUT) .disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS) .setDateFormat(new ISO8601DateFormat()) - .setSerializationInclusion(Include.NON_NULL) - .setDefaultMergeable(Boolean.TRUE); + .setSerializationInclusion(Include.NON_NULL); private static final String UDMI_VERSION_KEY = "UDMI_VERSION"; private static final String VERSION_KEY = "Version"; private static final String VERSION_MAIN_KEY = "main"; From bcd1c2b1d025c53f5e980339251c12cf9eab6f2b Mon Sep 17 00:00:00 2001 From: John Randolph Date: Wed, 6 Apr 2022 13:40:11 -0400 Subject: [PATCH 08/16] Simplify deepMerge() and remove unused functionality for list merging, which was not discussed. --- .../daq/mqtt/registrar/LocalDevice.java | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java b/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java index 81d1f47204..35be353cbc 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java @@ -263,22 +263,19 @@ public void validateExpected() { exceptionMap.throwIfNotEmpty(); } - private List listMerge(List list1, List list2) { - list2.removeAll(list1); - list1.addAll(list2); - return list1; - } private void deepMerge(Map map1, Map map2) { - for(String key : map2.keySet()) { + for (String key : map2.keySet()) { Object value2 = map2.get(key); if (map1.containsKey(key)) { Object value1 = map1.get(key); - if (value1 instanceof Map && value2 instanceof Map) + if (value1 instanceof Map && value2 instanceof Map) { deepMerge((Map) value1, (Map) value2); - else if (value1 instanceof List && value2 instanceof List) - map1.put(key, listMerge((List) value1, (List) value2)); - else map1.put(key, value2); - } else map1.put(key, value2); + } else { + map1.put(key, value2); + } + } else { + map1.put(key, value2); + } } } From 4abac218e84a6542f544c2021a156c747cae3cea Mon Sep 17 00:00:00 2001 From: John Randolph Date: Thu, 7 Apr 2022 11:56:24 -0400 Subject: [PATCH 09/16] Fixes based on checkstyle. --- .../daq/mqtt/registrar/LocalDevice.java | 55 ++++++++++++------- .../google/daq/mqtt/registrar/Registrar.java | 2 + 2 files changed, 37 insertions(+), 20 deletions(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java b/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java index 35be353cbc..eddea6f586 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java @@ -42,8 +42,17 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.*; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.Map.Entry; +import java.util.Objects; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeMap; +import java.util.TreeSet; import java.util.regex.Pattern; import org.apache.commons.io.IOUtils; import udmi.schema.Config; @@ -217,8 +226,8 @@ class LocalDevice { } LocalDevice( - File siteDir, File devicesDir, String deviceId, Map schemas, - String generation) { + File siteDir, File devicesDir, String deviceId, Map schemas, + String generation) { this(siteDir, devicesDir, deviceId, schemas, generation, null); } @@ -263,18 +272,21 @@ public void validateExpected() { exceptionMap.throwIfNotEmpty(); } - private void deepMerge(Map map1, Map map2) { + private void deepMergeDefaults(Map map1, Map map2) { for (String key : map2.keySet()) { Object value2 = map2.get(key); if (map1.containsKey(key)) { Object value1 = map1.get(key); + // Only deep copy maps, don't copy values from map2, or the defaults override new values. if (value1 instanceof Map && value2 instanceof Map) { - deepMerge((Map) value1, (Map) value2); - } else { - map1.put(key, value2); + deepMergeDefaults((Map) value1, (Map) value2); } } else { - map1.put(key, value2); + // Subtle. If map1[key] does not exist, copy the default value from map2[key] only if + // it is not a map. However this will only work for 1 level deep. + if (!(value2 instanceof Map)) { + map1.put(key, value2); + } } } } @@ -283,8 +295,8 @@ private Metadata readMetadata() { File metadataFile = new File(deviceDir, METADATA_JSON); try (InputStream targetStream = new FileInputStream(metadataFile)) { schemas.get(METADATA_JSON).validate(OBJECT_MAPPER.readTree(targetStream)); - } catch (ProcessingException | ValidationException metadata_exception) { - exceptionMap.put(EXCEPTION_VALIDATING, metadata_exception); + } catch (ProcessingException | ValidationException e) { + exceptionMap.put(EXCEPTION_VALIDATING, e); } catch (IOException ioException) { exceptionMap.put(EXCEPTION_LOADING, ioException); } @@ -292,12 +304,13 @@ private Metadata readMetadata() { if (siteDefaults == null) { return OBJECT_MAPPER.readValue(metadataFile, Metadata.class); } else { - final Map metadata_base = OBJECT_MAPPER.readValue(metadataFile, TreeMap.class); - deepMerge(metadata_base, siteDefaults); - return OBJECT_MAPPER.convertValue(metadata_base, Metadata.class); + final Map metadataBase = OBJECT_MAPPER.readValue(metadataFile, + TreeMap.class); + deepMergeDefaults(metadataBase, siteDefaults); + return OBJECT_MAPPER.convertValue(metadataBase, Metadata.class); } - } catch (Exception mapping_exception) { - exceptionMap.put(EXCEPTION_READING, mapping_exception); + } catch (Exception e) { + exceptionMap.put(EXCEPTION_READING, e); } return null; } @@ -306,7 +319,7 @@ private Metadata readNormalized() { try { File metadataFile = new File(outDir, NORMALIZED_JSON); return OBJECT_MAPPER.readValue(metadataFile, Metadata.class); - } catch (Exception mapping_exception) { + } catch (Exception e) { return new Metadata(); } } @@ -551,11 +564,11 @@ private PointsetConfig getDevicePointsetConfig() { metadata.pointset.points.forEach( (metadataKey, value) -> pointsetConfig.points.computeIfAbsent( - metadataKey, configKey -> ConfigFromMetadata(value))); + metadataKey, configKey -> configFromMetadata(value))); return pointsetConfig; } - PointPointsetConfig ConfigFromMetadata(PointPointsetMetadata metadata) { + PointPointsetConfig configFromMetadata(PointPointsetMetadata metadata) { PointPointsetConfig pointConfig = new PointPointsetConfig(); pointConfig.ref = metadata.ref; if (Boolean.TRUE.equals(metadata.writable)) { @@ -591,7 +604,8 @@ public void validateEnvelope(String registryId, String siteName) { envelope.projectId = fakeProjectId(); envelope.deviceNumId = makeNumId(envelope); String envelopeJson = OBJECT_MAPPER.writeValueAsString(envelope); - ProcessingReport processingReport = schemas.get(ENVELOPE_JSON).validate(OBJECT_MAPPER.readTree(envelopeJson)); + ProcessingReport processingReport = schemas.get(ENVELOPE_JSON) + .validate(OBJECT_MAPPER.readTree(envelopeJson)); if (!processingReport.isSuccess()) { processingReport.forEach(action -> { throw new RuntimeException("against schema", action.asException()); @@ -730,7 +744,7 @@ public void captureError(String exceptionType, Exception exception) { File exceptionLog = new File(outDir, EXCEPTION_LOG_FILE); try { try (FileWriter fileWriter = new FileWriter(exceptionLog, true); - PrintWriter printWriter = new PrintWriter(fileWriter)) { + PrintWriter printWriter = new PrintWriter(fileWriter)) { printWriter.println(exceptionType); exception.printStackTrace(printWriter); } @@ -782,6 +796,7 @@ public Metadata getMetadata() { } private static class ProperPrettyPrinterPolicy extends DefaultPrettyPrinter { + @Override public void writeObjectFieldValueSeparator(JsonGenerator jg) throws IOException { jg.writeRaw(": "); diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java b/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java index e665e2b010..ecad347ca3 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java @@ -666,6 +666,8 @@ private void loadSiteDefaults() { schemas.get(METADATA_JSON).validate(OBJECT_MAPPER.readTree(targetStream)); } catch (ProcessingException | ValidationException e) { throw new RuntimeException("While validating " + SITE_DEFAULTS_JSON, e); + } catch (FileNotFoundException e) { + return; } catch (IOException e) { throw new RuntimeException("While validating " + SITE_DEFAULTS_JSON, e); } From 2808ea4e09d3189ab4071b76d65fce7faf76c82e Mon Sep 17 00:00:00 2001 From: John Randolph Date: Thu, 7 Apr 2022 12:42:31 -0400 Subject: [PATCH 10/16] Restore copying entire maps from defaults to metadata. --- .../java/com/google/daq/mqtt/registrar/LocalDevice.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java b/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java index eddea6f586..134062c86b 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java @@ -282,11 +282,7 @@ private void deepMergeDefaults(Map map1, Map map deepMergeDefaults((Map) value1, (Map) value2); } } else { - // Subtle. If map1[key] does not exist, copy the default value from map2[key] only if - // it is not a map. However this will only work for 1 level deep. - if (!(value2 instanceof Map)) { - map1.put(key, value2); - } + map1.put(key, value2); } } } From fe587bfb0367cdd7321a6e3485e1481f1db095b1 Mon Sep 17 00:00:00 2001 From: John Randolph Date: Thu, 7 Apr 2022 12:46:26 -0400 Subject: [PATCH 11/16] Style cleanup --- .../google/daq/mqtt/registrar/Registrar.java | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java b/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java index ecad347ca3..3372aa823b 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java @@ -18,10 +18,19 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; -import com.google.daq.mqtt.util.*; +import com.google.daq.mqtt.util.CloudDeviceSettings; +import com.google.daq.mqtt.util.CloudIotManager; +import com.google.daq.mqtt.util.ConfigUtil; +import com.google.daq.mqtt.util.ExceptionMap; import com.google.daq.mqtt.util.ExceptionMap.ErrorTree; - -import java.io.*; +import com.google.daq.mqtt.util.PubSubPusher; +import com.google.daq.mqtt.util.ValidationException; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FilenameFilter; +import java.io.IOException; +import java.io.InputStream; import java.math.BigInteger; import java.net.URI; import java.time.Duration; @@ -597,7 +606,8 @@ private Map loadDevices(File siteDir, File devicesDir, LocalDevice localDevice = localDevices.computeIfAbsent( deviceName, - keyName -> new LocalDevice(siteDir, devicesDir, deviceName, schemas, generation, siteDefaults)); + keyName -> new LocalDevice(siteDir, devicesDir, deviceName, schemas, generation, + siteDefaults)); try { localDevice.loadCredentials(); } catch (Exception e) { @@ -658,8 +668,9 @@ private void loadSchema(String key) { private void loadSiteDefaults() { this.siteDefaults = null; - if (!schemas.containsKey(METADATA_JSON)) + if (!schemas.containsKey(METADATA_JSON)) { return; + } File siteDefaultsFile = new File(siteDir, SITE_DEFAULTS_JSON); try (InputStream targetStream = new FileInputStream(siteDefaultsFile)) { From d1186334414fe6cac5b4e2e673f6f765ca80d7f1 Mon Sep 17 00:00:00 2001 From: John Randolph Date: Thu, 7 Apr 2022 17:45:28 -0400 Subject: [PATCH 12/16] Refactor argument names for deepMergeDefaults. Lazier exception catching. --- .../google/daq/mqtt/registrar/LocalDevice.java | 15 ++++++++------- .../com/google/daq/mqtt/registrar/Registrar.java | 4 +--- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java b/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java index 134062c86b..7bbd377d3f 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java @@ -272,17 +272,18 @@ public void validateExpected() { exceptionMap.throwIfNotEmpty(); } - private void deepMergeDefaults(Map map1, Map map2) { - for (String key : map2.keySet()) { - Object value2 = map2.get(key); - if (map1.containsKey(key)) { - Object value1 = map1.get(key); - // Only deep copy maps, don't copy values from map2, or the defaults override new values. + private void deepMergeDefaults(Map destination, Map source) { + for (String key : source.keySet()) { + Object value2 = source.get(key); + if (destination.containsKey(key)) { + Object value1 = destination.get(key); + // When destination and source both contain key, deep copy maps but not other key/values, + // which would produce config override rather than defaults. if (value1 instanceof Map && value2 instanceof Map) { deepMergeDefaults((Map) value1, (Map) value2); } } else { - map1.put(key, value2); + destination.put(key, value2); } } } diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java b/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java index 3372aa823b..dd2752d52d 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java @@ -675,11 +675,9 @@ private void loadSiteDefaults() { File siteDefaultsFile = new File(siteDir, SITE_DEFAULTS_JSON); try (InputStream targetStream = new FileInputStream(siteDefaultsFile)) { schemas.get(METADATA_JSON).validate(OBJECT_MAPPER.readTree(targetStream)); - } catch (ProcessingException | ValidationException e) { - throw new RuntimeException("While validating " + SITE_DEFAULTS_JSON, e); } catch (FileNotFoundException e) { return; - } catch (IOException e) { + } catch (Exception e) { throw new RuntimeException("While validating " + SITE_DEFAULTS_JSON, e); } From 98ad968c55be618def6069104df5f6d84ee4c7fc Mon Sep 17 00:00:00 2001 From: John Randolph Date: Fri, 8 Apr 2022 11:18:55 -0400 Subject: [PATCH 13/16] Refactor "site defaults" to "site metadata". --- .../daq/mqtt/registrar/LocalDevice.java | 14 +++++------ .../google/daq/mqtt/registrar/Registrar.java | 25 ++++++++----------- 2 files changed, 18 insertions(+), 21 deletions(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java b/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java index 7bbd377d3f..337227e53d 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java @@ -196,7 +196,7 @@ class LocalDevice { private final ExceptionMap exceptionMap; private final String generation; private final List deviceCredentials = new ArrayList<>(); - private final TreeMap siteDefaults; + private final TreeMap siteMetadata; private String deviceNumId; @@ -204,16 +204,16 @@ class LocalDevice { LocalDevice( File siteDir, File devicesDir, String deviceId, Map schemas, - String generation, Metadata siteDefaults) { + String generation, Metadata siteMetadata) { try { this.deviceId = deviceId; this.schemas = schemas; this.generation = generation; this.siteDir = siteDir; - if (siteDefaults != null) { - this.siteDefaults = OBJECT_MAPPER.convertValue(siteDefaults, TreeMap.class); + if (siteMetadata != null) { + this.siteMetadata = OBJECT_MAPPER.convertValue(siteMetadata, TreeMap.class); } else { - this.siteDefaults = null; + this.siteMetadata = null; } exceptionMap = new ExceptionMap("Exceptions for " + deviceId); deviceDir = new File(devicesDir, deviceId); @@ -298,12 +298,12 @@ private Metadata readMetadata() { exceptionMap.put(EXCEPTION_LOADING, ioException); } try { - if (siteDefaults == null) { + if (siteMetadata == null) { return OBJECT_MAPPER.readValue(metadataFile, Metadata.class); } else { final Map metadataBase = OBJECT_MAPPER.readValue(metadataFile, TreeMap.class); - deepMergeDefaults(metadataBase, siteDefaults); + deepMergeDefaults(metadataBase, siteMetadata); return OBJECT_MAPPER.convertValue(metadataBase, Metadata.class); } } catch (Exception e) { diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java b/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java index dd2752d52d..d0e7162257 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java @@ -7,7 +7,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.databind.util.ISO8601DateFormat; -import com.github.fge.jsonschema.core.exceptions.ProcessingException; import com.github.fge.jsonschema.core.load.configuration.LoadingConfiguration; import com.github.fge.jsonschema.core.load.download.URIDownloader; import com.github.fge.jsonschema.main.JsonSchema; @@ -24,12 +23,10 @@ import com.google.daq.mqtt.util.ExceptionMap; import com.google.daq.mqtt.util.ExceptionMap.ErrorTree; import com.google.daq.mqtt.util.PubSubPusher; -import com.google.daq.mqtt.util.ValidationException; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FilenameFilter; -import java.io.IOException; import java.io.InputStream; import java.math.BigInteger; import java.net.URI; @@ -78,7 +75,7 @@ public class Registrar { private static final String SCHEMA_SUFFIX = ".json"; private static final String REGISTRATION_SUMMARY_JSON = "registration_summary.json"; private static final String SCHEMA_NAME = "UDMI"; - private static final String SITE_DEFAULTS_JSON = "site_defaults.json"; + private static final String SITE_METADATA_JSON = "site_metadata.json"; private static final String SWARM_SUBFOLDER = "swarm"; private static final long PROCESSING_TIMEOUT_MIN = 60; private final Map schemas = new HashMap<>(); @@ -95,7 +92,7 @@ public class Registrar { private boolean updateCloudIoT; private Duration idleLimit; private Set cloudDevices; - private Metadata siteDefaults; + private Metadata siteMetadata; public static void main(String[] args) { ArrayList argList = new ArrayList<>(List.of(args)); @@ -107,7 +104,7 @@ public static void main(String[] args) { registrar.setToolRoot(null); } - registrar.loadSiteDefaults(); + registrar.loadSiteMetadata(); if (processAllDevices) { registrar.processDevices(); @@ -607,7 +604,7 @@ private Map loadDevices(File siteDir, File devicesDir, localDevices.computeIfAbsent( deviceName, keyName -> new LocalDevice(siteDir, devicesDir, deviceName, schemas, generation, - siteDefaults)); + siteMetadata)); try { localDevice.loadCredentials(); } catch (Exception e) { @@ -665,26 +662,26 @@ private void loadSchema(String key) { } } - private void loadSiteDefaults() { - this.siteDefaults = null; + private void loadSiteMetadata() { + this.siteMetadata = null; if (!schemas.containsKey(METADATA_JSON)) { return; } - File siteDefaultsFile = new File(siteDir, SITE_DEFAULTS_JSON); - try (InputStream targetStream = new FileInputStream(siteDefaultsFile)) { + File siteMetadataFile = new File(siteDir, SITE_METADATA_JSON); + try (InputStream targetStream = new FileInputStream(siteMetadataFile)) { schemas.get(METADATA_JSON).validate(OBJECT_MAPPER.readTree(targetStream)); } catch (FileNotFoundException e) { return; } catch (Exception e) { - throw new RuntimeException("While validating " + SITE_DEFAULTS_JSON, e); + throw new RuntimeException("While validating " + SITE_METADATA_JSON, e); } try { - this.siteDefaults = OBJECT_MAPPER.readValue(siteDefaultsFile, Metadata.class); + this.siteMetadata = OBJECT_MAPPER.readValue(siteMetadataFile, Metadata.class); } catch (Exception e) { - throw new RuntimeException("While loading " + SITE_DEFAULTS_JSON, e); + throw new RuntimeException("While loading " + SITE_METADATA_JSON, e); } } From 86ab3d8fac467945cdc69357dee7d5deda191841 Mon Sep 17 00:00:00 2001 From: John Randolph Date: Tue, 12 Apr 2022 17:29:12 -0400 Subject: [PATCH 14/16] Add tests to test_registrar --- bin/test_registrar | 38 ++++++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/bin/test_registrar b/bin/test_registrar index c68525523a..74bf981018 100755 --- a/bin/test_registrar +++ b/bin/test_registrar @@ -1,19 +1,41 @@ #!/bin/bash -e ROOT_DIR=$(dirname $0)/.. -cd $ROOT_DIR +cd ${ROOT_DIR} bin/clone_model TEST_SITE=udmi_site_model -bin/registrar $TEST_SITE +bin/registrar ${TEST_SITE} -cat $TEST_SITE/registration_summary.json +cat "${TEST_SITE}/registration_summary.json" echo -devices=$(fgrep 'Z"' $TEST_SITE/registration_summary.json | wc -l) - -echo Found $devices clean devices. - -[ "$devices" == 4 ] +devices=$(fgrep 'Z"' ${TEST_SITE}/registration_summary.json | wc -l) + +# Test site_metadata settings for system.location.site. +site=$(jq -r .system.location.site < ${TEST_SITE}/site_metadata.json) + +sm_devices=0 +for name in ${TEST_SITE}/devices/* ; do + if [[ -f ${name}/out/metadata_norm.json ]]; then + supplied_site=$(jq -r ".system.location.site" < ${name}/metadata.json) + # If no site value is supplied in per-device metadata, expect default. + if [[ "${supplied_site}" == "null" ]]; then + jq -e ".system.location.site == \"${site}\"" \ + ${name}/out/metadata_norm.json > /dev/null \ + && sm_devices=$[sm_devices+1] + else + jq -e ".system.location.site == \"${supplied_site}\"" \ + ${name}/out/metadata_norm.json > /dev/null \ + && sm_devices=$[sm_devices+1] + fi + fi +done + +echo Found ${devices} clean devices. +echo Found ${sm_devices} devices with correct site_metadata values. + +[ "${devices}" == 4 ] +[ "${sm_devices}" == "${devices}" ] From d7522bc4811cb55ea1037ef1ffcc56ec5715a635 Mon Sep 17 00:00:00 2001 From: John Randolph Date: Thu, 14 Apr 2022 15:04:57 -0400 Subject: [PATCH 15/16] Add print output when site_metadata is loaded. --- .../src/main/java/com/google/daq/mqtt/registrar/Registrar.java | 1 + 1 file changed, 1 insertion(+) diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java b/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java index d0e7162257..3b36a3fa2f 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java @@ -679,6 +679,7 @@ private void loadSiteMetadata() { } try { + System.err.printf("Loading " + SITE_METADATA_JSON + "\n"); this.siteMetadata = OBJECT_MAPPER.readValue(siteMetadataFile, Metadata.class); } catch (Exception e) { throw new RuntimeException("While loading " + SITE_METADATA_JSON, e); From 3ebadb0a6fa5ee72b7da46f4cd14bbc318e0dde8 Mon Sep 17 00:00:00 2001 From: John Randolph Date: Thu, 14 Apr 2022 15:18:12 -0400 Subject: [PATCH 16/16] Run correctly whether site_metadata.json is supplied or not. --- bin/test_registrar | 51 ++++++++++++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/bin/test_registrar b/bin/test_registrar index 74bf981018..5b8ec7d606 100755 --- a/bin/test_registrar +++ b/bin/test_registrar @@ -14,28 +14,35 @@ echo devices=$(fgrep 'Z"' ${TEST_SITE}/registration_summary.json | wc -l) -# Test site_metadata settings for system.location.site. -site=$(jq -r .system.location.site < ${TEST_SITE}/site_metadata.json) - -sm_devices=0 -for name in ${TEST_SITE}/devices/* ; do - if [[ -f ${name}/out/metadata_norm.json ]]; then - supplied_site=$(jq -r ".system.location.site" < ${name}/metadata.json) - # If no site value is supplied in per-device metadata, expect default. - if [[ "${supplied_site}" == "null" ]]; then - jq -e ".system.location.site == \"${site}\"" \ - ${name}/out/metadata_norm.json > /dev/null \ - && sm_devices=$[sm_devices+1] - else - jq -e ".system.location.site == \"${supplied_site}\"" \ - ${name}/out/metadata_norm.json > /dev/null \ - && sm_devices=$[sm_devices+1] - fi - fi -done +exit_status=0 echo Found ${devices} clean devices. -echo Found ${sm_devices} devices with correct site_metadata values. +[ "${devices}" == 4 ] || exit_status=1 + +if [[ -f ${TEST_SITE}/site_metadata.json ]]; then + + # Test site_metadata settings for system.location.site. + site=$(jq -r .system.location.site < ${TEST_SITE}/site_metadata.json) + + sm_devices=0 + for name in ${TEST_SITE}/devices/* ; do + if [[ -f ${name}/out/metadata_norm.json ]]; then + supplied_site=$(jq -r ".system.location.site" < ${name}/metadata.json) + # If no site value is supplied in per-device metadata, expect default. + if [[ "${supplied_site}" == "null" ]]; then + jq -e ".system.location.site == \"${site}\"" \ + ${name}/out/metadata_norm.json > /dev/null \ + && sm_devices=$[sm_devices+1] + else + jq -e ".system.location.site == \"${supplied_site}\"" \ + ${name}/out/metadata_norm.json > /dev/null \ + && sm_devices=$[sm_devices+1] + fi + fi + done + + echo Found ${sm_devices} devices with correct site_metadata values. + [ "${sm_devices}" == "${devices}" ] || exit_status=1 +fi -[ "${devices}" == 4 ] -[ "${sm_devices}" == "${devices}" ] +exit $exit_status