Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add site_defaults.json in site model directory, which can supply default values to all Metadata. #288

Merged
merged 24 commits into from
Apr 14, 2022
Merged

Add site_defaults.json in site model directory, which can supply default values to all Metadata. #288

Show file tree
Hide file tree
Changes from 10 commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
a713dc0
Add site_defaults so that sample_rate_sec, sample_limit_sec, min_logl…
johnrandolph Mar 31, 2022
87a67b7
Load site_defaults.json into site-wide Metadata. Merge values from
johnrandolph Apr 6, 2022
cd94f71
Merge branch 'faucetsdn:master' into site_wide_defaults
johnrandolph Apr 6, 2022
cfc48f0
Set null instead of making a map instance.
johnrandolph Apr 6, 2022
1f6c018
Merge branch 'site_wide_defaults' of github.com:johnrandolph/udmi int…
johnrandolph Apr 6, 2022
93545a6
Remove site_defaults schema which is no longer needed, because
johnrandolph Apr 6, 2022
bed4cff
Restore readMetadata() and remove intermediate readMetadataBase(),
johnrandolph Apr 6, 2022
4fa3f81
Cleanup in Registrar loadSiteDefaults().
johnrandolph Apr 6, 2022
aeb0ac9
Remove setDefaultMergeable() property set, an earlier attempt to
johnrandolph Apr 6, 2022
bcd1c2b
Simplify deepMerge() and remove unused functionality for list merging,
johnrandolph Apr 6, 2022
907cd59
Merge remote-tracking branch 'origin/master' into site_wide_defaults
johnrandolph Apr 7, 2022
4abac21
Fixes based on checkstyle.
johnrandolph Apr 7, 2022
2808ea4
Restore copying entire maps from defaults to metadata.
johnrandolph Apr 7, 2022
fe587bf
Style cleanup
johnrandolph Apr 7, 2022
d118633
Refactor argument names for deepMergeDefaults.
johnrandolph Apr 7, 2022
98ad968
Refactor "site defaults" to "site metadata".
johnrandolph Apr 8, 2022
2b5f8e5
Merge branch 'faucetsdn:master' into site_wide_defaults
johnrandolph Apr 8, 2022
926d360
Merge branch 'faucetsdn:master' into site_wide_defaults
johnrandolph Apr 8, 2022
86ab3d8
Add tests to test_registrar
johnrandolph Apr 12, 2022
d7522bc
Add print output when site_metadata is loaded.
johnrandolph Apr 14, 2022
d648612
Merge branch 'faucetsdn:master' into site_wide_defaults
johnrandolph Apr 14, 2022
3ebadb0
Run correctly whether site_metadata.json is supplied or not.
johnrandolph Apr 14, 2022
3f3a05a
Merge branch 'faucetsdn:master' into site_wide_defaults
johnrandolph Apr 14, 2022
6cd09c0
Merge branch 'site_wide_defaults' of github.com:johnrandolph/udmi int…
johnrandolph Apr 14, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 80 additions & 41 deletions validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,16 +42,8 @@
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
import java.util.regex.Pattern;
import org.apache.commons.io.IOUtils;
import udmi.schema.Config;
Expand All @@ -67,17 +59,8 @@

class LocalDevice {

public static final String INVALID_METADATA_HASH = "INVALID";
public static final String EXCEPTION_VALIDATING = "Validating";
public static final String EXCEPTION_LOADING = "Loading";
public static final String EXCEPTION_READING = "Reading";
public static final String EXCEPTION_WRITING = "Writing";
public static final String EXCEPTION_FILES = "Files";
public static final String EXCEPTION_REGISTERING = "Registering";
public static final String EXCEPTION_CREDENTIALS = "Credential";
public static final String EXCEPTION_ENVELOPE = "Envelope";
public static final String EXCEPTION_SAMPLES = "Samples";
private static final PrettyPrinter PROPER_PRETTY_PRINTER_POLICY = new ProperPrettyPrinterPolicy();

private static final ObjectMapper OBJECT_MAPPER_RAW =
new ObjectMapper()
.enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS)
Expand All @@ -86,11 +69,13 @@ class LocalDevice {
.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS)
.setDateFormat(new ISO8601DateFormat())
.setSerializationInclusion(Include.NON_NULL);

private static final ObjectMapper OBJECT_MAPPER =
OBJECT_MAPPER_RAW
.copy()
.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
.enable(SerializationFeature.INDENT_OUTPUT);

private static final String RSA_AUTH_TYPE = "RS256";
private static final String RSA_CERT_TYPE = "RS256_X509";
private static final String RSA_KEY_FORMAT = "RSA_PEM";
Expand All @@ -101,6 +86,7 @@ class LocalDevice {
private static final String RSA_CERT_PEM = "rsa_cert.pem";
private static final String RSA_PRIVATE_PEM = "rsa_private.pem";
private static final String RSA_PRIVATE_PKCS8 = "rsa_private.pkcs8";

private static final String ES_AUTH_TYPE = "ES256";
private static final String ES_CERT_TYPE = "ES256_X509";
private static final String ES_KEY_FORMAT = "ES256_PEM";
Expand All @@ -111,17 +97,14 @@ class LocalDevice {
private static final String ES_CERT_PEM = "ec_cert.pem";
private static final String ES_PRIVATE_PEM = "ec_private.pem";
private static final String ES_PRIVATE_PKCS8 = "ec_private.pkcs8";
protected static final Map<String, String> PRIVATE_PKCS8_MAP =
ImmutableMap.of(
RSA_AUTH_TYPE, RSA_PRIVATE_PKCS8,
RSA_CERT_TYPE, RSA_PRIVATE_PKCS8,
ES_AUTH_TYPE, ES_PRIVATE_PKCS8,
ES_CERT_TYPE, ES_PRIVATE_PKCS8);

private static final String SAMPLES_DIR = "samples";
private static final String AUX_DIR = "aux";
private static final String OUT_DIR = "out";
private static final String EXCEPTION_LOG_FILE = "exceptions.txt";

private static final Set<String> DEVICE_FILES = ImmutableSet.of(METADATA_JSON);

private static final Set<String> RSA_PRIVATE_KEY_FILES =
ImmutableSet.of(RSA_PRIVATE_PEM, RSA_PRIVATE_PKCS8);
private static final Set<String> ES_PRIVATE_KEY_FILES =
Expand All @@ -132,6 +115,14 @@ class LocalDevice {
RSA_CERT_TYPE, RSA_PRIVATE_KEY_FILES,
ES_AUTH_TYPE, ES_PRIVATE_KEY_FILES,
ES_CERT_TYPE, ES_PRIVATE_KEY_FILES);

protected static final Map<String, String> PRIVATE_PKCS8_MAP =
ImmutableMap.of(
RSA_AUTH_TYPE, RSA_PRIVATE_PKCS8,
RSA_CERT_TYPE, RSA_PRIVATE_PKCS8,
ES_AUTH_TYPE, ES_PRIVATE_PKCS8,
ES_CERT_TYPE, ES_PRIVATE_PKCS8);

private static final Map<String, String> PUBLIC_KEY_FILE_MAP =
ImmutableMap.of(
RSA_AUTH_TYPE, RSA_PUBLIC_PEM,
Expand All @@ -151,8 +142,10 @@ class LocalDevice {
SAMPLES_DIR,
AUX_DIR,
OUT_DIR);

private static final Set<String> OUT_FILES =
ImmutableSet.of(GENERATED_CONFIG_JSON, DEVICE_ERRORS_JSON, NORMALIZED_JSON);

private static final Set<String> ALL_KEY_FILES =
ImmutableSet.of(
RSA_PUBLIC_PEM,
Expand All @@ -168,9 +161,23 @@ class LocalDevice {
RSA_CERT_TYPE, RSA_CERT_FORMAT,
ES_AUTH_TYPE, ES_KEY_FORMAT,
ES_CERT_TYPE, ES_CERT_FILE);

private static final String ERROR_FORMAT_INDENT = " ";
private static final int MAX_METADATA_LENGTH = 32767;
public static final String UDMI_VERSION = "1.3.14";
private static final String UDMI_VERSION = "1.3.14";

public static final String INVALID_METADATA_HASH = "INVALID";

public static final String EXCEPTION_VALIDATING = "Validating";
public static final String EXCEPTION_LOADING = "Loading";
public static final String EXCEPTION_READING = "Reading";
public static final String EXCEPTION_WRITING = "Writing";
public static final String EXCEPTION_FILES = "Files";
public static final String EXCEPTION_REGISTERING = "Registering";
public static final String EXCEPTION_CREDENTIALS = "Credential";
public static final String EXCEPTION_ENVELOPE = "Envelope";
public static final String EXCEPTION_SAMPLES = "Samples";

private final String deviceId;
private final Map<String, JsonSchema> schemas;
private final File siteDir;
Expand All @@ -180,19 +187,25 @@ class LocalDevice {
private final ExceptionMap exceptionMap;
private final String generation;
private final List<DeviceCredential> deviceCredentials = new ArrayList<>();
private final TreeMap<String, Object> siteDefaults;

private String deviceNumId;

private CloudDeviceSettings settings;

LocalDevice(
File siteDir, File devicesDir, String deviceId, Map<String, JsonSchema> schemas,
String generation) {
String generation, Metadata siteDefaults) {
try {
this.deviceId = deviceId;
this.schemas = schemas;
this.generation = generation;
this.siteDir = siteDir;
if (siteDefaults != null) {
this.siteDefaults = OBJECT_MAPPER.convertValue(siteDefaults, TreeMap.class);
} else {
this.siteDefaults = null;
}
exceptionMap = new ExceptionMap("Exceptions for " + deviceId);
deviceDir = new File(devicesDir, deviceId);
outDir = new File(deviceDir, OUT_DIR);
Expand All @@ -203,8 +216,10 @@ class LocalDevice {
}
}

static boolean deviceExists(File devicesDir, String deviceName) {
return new File(new File(devicesDir, deviceName), METADATA_JSON).isFile();
LocalDevice(
File siteDir, File devicesDir, String deviceId, Map<String, JsonSchema> schemas,
String generation) {
this(siteDir, devicesDir, deviceId, schemas, generation, null);
}

private void prepareOutDir() {
Expand All @@ -215,6 +230,10 @@ private void prepareOutDir() {
exceptionLog.delete();
}

static boolean deviceExists(File devicesDir, String deviceName) {
return new File(new File(devicesDir, deviceName), METADATA_JSON).isFile();
}

public void validateExpected() {
Path relativized = siteDir.toPath().relativize(deviceDir.toPath());
ExceptionMap exceptionMap = new ExceptionMap(relativized.toString());
Expand Down Expand Up @@ -244,19 +263,41 @@ public void validateExpected() {
exceptionMap.throwIfNotEmpty();
}

private void deepMerge(Map<String, Object> map1, Map<String, Object> map2) {
johnrandolph marked this conversation as resolved.
Show resolved Hide resolved
for (String key : map2.keySet()) {
Object value2 = map2.get(key);
if (map1.containsKey(key)) {
Object value1 = map1.get(key);
if (value1 instanceof Map && value2 instanceof Map) {
deepMerge((Map<String, Object>) value1, (Map<String, Object>) value2);
} else {
map1.put(key, value2);
}
} else {
map1.put(key, value2);
}
}
}

private Metadata readMetadata() {
File metadataFile = new File(deviceDir, METADATA_JSON);
try (InputStream targetStream = new FileInputStream(metadataFile)) {
schemas.get(METADATA_JSON).validate(OBJECT_MAPPER.readTree(targetStream));
} catch (ProcessingException | ValidationException metadataException) {
exceptionMap.put(EXCEPTION_VALIDATING, metadataException);
} catch (ProcessingException | ValidationException metadata_exception) {
exceptionMap.put(EXCEPTION_VALIDATING, metadata_exception);
} catch (IOException ioException) {
exceptionMap.put(EXCEPTION_LOADING, ioException);
}
try {
return OBJECT_MAPPER.readValue(metadataFile, Metadata.class);
} catch (Exception mappingException) {
exceptionMap.put(EXCEPTION_READING, mappingException);
if (siteDefaults == null) {
return OBJECT_MAPPER.readValue(metadataFile, Metadata.class);
} else {
final Map<String, Object> metadata_base = OBJECT_MAPPER.readValue(metadataFile, TreeMap.class);
deepMerge(metadata_base, siteDefaults);
return OBJECT_MAPPER.convertValue(metadata_base, Metadata.class);
}
} catch (Exception mapping_exception) {
exceptionMap.put(EXCEPTION_READING, mapping_exception);
}
return null;
}
Expand All @@ -265,7 +306,7 @@ private Metadata readNormalized() {
try {
File metadataFile = new File(outDir, NORMALIZED_JSON);
return OBJECT_MAPPER.readValue(metadataFile, Metadata.class);
} catch (Exception mappingException) {
} catch (Exception mapping_exception) {
return new Metadata();
}
}
Expand Down Expand Up @@ -510,11 +551,11 @@ private PointsetConfig getDevicePointsetConfig() {
metadata.pointset.points.forEach(
(metadataKey, value) ->
pointsetConfig.points.computeIfAbsent(
metadataKey, configKey -> configFromMetadata(value)));
metadataKey, configKey -> ConfigFromMetadata(value)));
return pointsetConfig;
}

PointPointsetConfig configFromMetadata(PointPointsetMetadata metadata) {
PointPointsetConfig ConfigFromMetadata(PointPointsetMetadata metadata) {
PointPointsetConfig pointConfig = new PointPointsetConfig();
pointConfig.ref = metadata.ref;
if (Boolean.TRUE.equals(metadata.writable)) {
Expand Down Expand Up @@ -550,8 +591,7 @@ public void validateEnvelope(String registryId, String siteName) {
envelope.projectId = fakeProjectId();
envelope.deviceNumId = makeNumId(envelope);
String envelopeJson = OBJECT_MAPPER.writeValueAsString(envelope);
ProcessingReport processingReport = schemas.get(ENVELOPE_JSON)
.validate(OBJECT_MAPPER.readTree(envelopeJson));
ProcessingReport processingReport = schemas.get(ENVELOPE_JSON).validate(OBJECT_MAPPER.readTree(envelopeJson));
if (!processingReport.isSuccess()) {
processingReport.forEach(action -> {
throw new RuntimeException("against schema", action.asException());
Expand Down Expand Up @@ -690,7 +730,7 @@ public void captureError(String exceptionType, Exception exception) {
File exceptionLog = new File(outDir, EXCEPTION_LOG_FILE);
try {
try (FileWriter fileWriter = new FileWriter(exceptionLog, true);
PrintWriter printWriter = new PrintWriter(fileWriter)) {
PrintWriter printWriter = new PrintWriter(fileWriter)) {
printWriter.println(exceptionType);
exception.printStackTrace(printWriter);
}
Expand Down Expand Up @@ -742,7 +782,6 @@ public Metadata getMetadata() {
}

private static class ProperPrettyPrinterPolicy extends DefaultPrettyPrinter {

@Override
public void writeObjectFieldValueSeparator(JsonGenerator jg) throws IOException {
jg.writeRaw(": ");
Expand Down
42 changes: 32 additions & 10 deletions validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.databind.util.ISO8601DateFormat;
import com.github.fge.jsonschema.core.exceptions.ProcessingException;
import com.github.fge.jsonschema.core.load.configuration.LoadingConfiguration;
import com.github.fge.jsonschema.core.load.download.URIDownloader;
import com.github.fge.jsonschema.main.JsonSchema;
Expand All @@ -17,16 +18,10 @@
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import com.google.daq.mqtt.util.CloudDeviceSettings;
import com.google.daq.mqtt.util.CloudIotManager;
import com.google.daq.mqtt.util.ConfigUtil;
import com.google.daq.mqtt.util.ExceptionMap;
import com.google.daq.mqtt.util.*;
import com.google.daq.mqtt.util.ExceptionMap.ErrorTree;
import com.google.daq.mqtt.util.PubSubPusher;
import java.io.File;
import java.io.FileInputStream;
import java.io.FilenameFilter;
import java.io.InputStream;

import java.io.*;
import java.math.BigInteger;
import java.net.URI;
import java.time.Duration;
Expand All @@ -49,6 +44,7 @@
import java.util.stream.Collectors;
import udmi.schema.Config;
import udmi.schema.Envelope.SubFolder;
import udmi.schema.Metadata;

public class Registrar {

Expand All @@ -73,6 +69,7 @@ public class Registrar {
private static final String SCHEMA_SUFFIX = ".json";
private static final String REGISTRATION_SUMMARY_JSON = "registration_summary.json";
private static final String SCHEMA_NAME = "UDMI";
private static final String SITE_DEFAULTS_JSON = "site_defaults.json";
grafnu marked this conversation as resolved.
Show resolved Hide resolved
private static final String SWARM_SUBFOLDER = "swarm";
private static final long PROCESSING_TIMEOUT_MIN = 60;
private final Map<String, JsonSchema> schemas = new HashMap<>();
Expand All @@ -89,6 +86,7 @@ public class Registrar {
private boolean updateCloudIoT;
private Duration idleLimit;
private Set<String> cloudDevices;
private Metadata siteDefaults;

public static void main(String[] args) {
ArrayList<String> argList = new ArrayList<>(List.of(args));
Expand All @@ -100,6 +98,8 @@ public static void main(String[] args) {
registrar.setToolRoot(null);
}

registrar.loadSiteDefaults();

if (processAllDevices) {
registrar.processDevices();
} else {
Expand Down Expand Up @@ -597,7 +597,7 @@ private Map<String, LocalDevice> loadDevices(File siteDir, File devicesDir,
LocalDevice localDevice =
localDevices.computeIfAbsent(
deviceName,
keyName -> new LocalDevice(siteDir, devicesDir, deviceName, schemas, generation));
keyName -> new LocalDevice(siteDir, devicesDir, deviceName, schemas, generation, siteDefaults));
try {
localDevice.loadCredentials();
} catch (Exception e) {
Expand Down Expand Up @@ -655,6 +655,28 @@ private void loadSchema(String key) {
}
}

private void loadSiteDefaults() {
this.siteDefaults = null;

if (!schemas.containsKey(METADATA_JSON))
return;

File siteDefaultsFile = new File(siteDir, SITE_DEFAULTS_JSON);
try (InputStream targetStream = new FileInputStream(siteDefaultsFile)) {
schemas.get(METADATA_JSON).validate(OBJECT_MAPPER.readTree(targetStream));
} catch (ProcessingException | ValidationException e) {
johnrandolph marked this conversation as resolved.
Show resolved Hide resolved
throw new RuntimeException("While validating " + SITE_DEFAULTS_JSON, e);
} catch (IOException e) {
throw new RuntimeException("While validating " + SITE_DEFAULTS_JSON, e);
}

try {
this.siteDefaults = OBJECT_MAPPER.readValue(siteDefaultsFile, Metadata.class);
} catch (Exception e) {
throw new RuntimeException("While loading " + SITE_DEFAULTS_JSON, e);
}
}

class RelativeDownloader implements URIDownloader {

@Override
Expand Down