Mapping Engine and Agent (#435)

.gencode_hash.txt

@@ -1,13 +1,13 @@
 52e5e736587cb4d9615b5b48dda53d385fb4c7091864e5d7ebaf804a46519852  gencode/docs/cloud_iot_config.html
 36842bbf9d023a83a3e8958fc3ae607424f4ada5b1e72be35fe2ac443cfd31cc  gencode/docs/command_discovery.html
-4f5f58997cc9f95ce7f7c251db954fad3e38f5579ce943922d3911d0e55cd57a  gencode/docs/command_mapping.html
+4d65d7043d4e37e7f043a3e6d4d859bc84cf6ed1c5a07d9f04b731ee9e6b16ef  gencode/docs/command_mapping.html
 d755727827955c38c6679869a4e9177e44a83770a1c6a046f953954792a6d4a4  gencode/docs/config.html
 6cffb39b5071680f39a7b96bdeb8330b6a8f5fa71ed4c4d42b82d7896080dcd9  gencode/docs/config_mapping.html
 62a5f579661625c0fa53320557be47711e289566b14bd0399b45395cea87ae27  gencode/docs/config_pointset.html
 34e05b6966c611afd4591bdb2f1a606123326c922b0747a3b0861383d68c1177  gencode/docs/configuration_endpoint.html
 01ee911c3d85ce9d5de98a77d15a78e7719bc151efdf5149082c8056b7a60bc1  gencode/docs/envelope.html
 b8972d7b07d1dc6bdb82988e1e9955603a80477cac18e7745ecbd7fe78c104ba  gencode/docs/event_discovery.html
-71c21b380bec6723e62f9f0c44b085a5d7c49f29a3f04d01cd93bed64061169e  gencode/docs/event_mapping.html
+971c2d352004e53c8fe4fa9370685dca8ed974a97b2e1e2382653be344010346  gencode/docs/event_mapping.html
 8133e380e40f27c56accbffc665b2eeb56ec84a4da3b52ba7aa5e439c9c40572  gencode/docs/event_pointset.html
 ac05c443556b92de96ec1867e72c668da27bc53012fe5821a8035a81ed46aaaf  gencode/docs/event_system.html
 51cc24f8ebe5daa9f165ec401cf4745994823037ba6a0753730339ef81a57fa6  gencode/docs/event_validation.html
@@ -27,8 +27,7 @@ d39d7fe37a41c74a40080af7b0a429d201ab1fdff7444428c4b98eb7b38c332b  gencode/java/u
 a7c57d119adcd0cf6363cc5301ce562004222522242e8ffd1d0cd7010f235ae1  gencode/java/udmi/schema/BlobBlobsetState.java
 b9f903444ab08907e41eb123286434ff3207b1edd01397af3ddefb8475bbdadc  gencode/java/udmi/schema/BlobsetConfig.java
 fcbed49f1af8b791d8c52bcbe18f65521a79d9ac3eb33ec3afd9b342ab2bfc56  gencode/java/udmi/schema/BlobsetState.java
-3e5f6f89b08a1174b5786eb20fc781b36c81db6ce999b970744a0f9644c6d3a5  gencode/java/udmi/schema/BuildingConfig.java
-ae7a42eadbd26d66d4d168ac090b933598c90b126346c0a72f6b82705c6324ce  gencode/java/udmi/schema/BuildingConfigTranslation.java
+0a4f6bcd5065418c1cdc6c05b900b3de31744847d25b6ab6de7aabb1e724710e  gencode/java/udmi/schema/BuildingTranslation.java
 5655b896f000b0108f438f9b04bfc092764ad5af63b70bbf750ba58b5f5527fa  gencode/java/udmi/schema/Category.java
 d6875f63ce67d1b945a0b75a4a660bd083cc52492371a7350c4109f6bf54968b  gencode/java/udmi/schema/CloudIotConfig.java
 a2eeff86f4302272736d84602e2ca36a64d27c8ef6761cc05ffb8ad17b030d4d  gencode/java/udmi/schema/CloudModel.java
@@ -57,7 +56,9 @@ a5e5adfc187709e8646a11c92e804acfb67743f9d72149008aaca954df3177f6  gencode/java/u
 07fd4911363437b274c19b024759b04b116152176702da8d4203c4ff4cb55b7f  gencode/java/udmi/schema/LocalnetConfig.java
 910c68183db7703b00bcb81146ad73e6fe0d4bbc4caec4dc9c621f3cc2e5eee5  gencode/java/udmi/schema/LocalnetModel.java
 2df4ae32d0bbecc21f7c3f6a416a195baa766a6210cfa8abca4a7bb45b9c7961  gencode/java/udmi/schema/Location.java
+b815ba1f198fd32b11fbedf71e5fa820bba08713dc5c95603c700dfa21ec5904  gencode/java/udmi/schema/MappingCommand.java
 0f6f85a4aaffb90735acc2565e8e4afa349d3ded5781cb1dce1ebbaa237feaa7  gencode/java/udmi/schema/MappingConfig.java
+7f790aa98c03fcd00826a48fcf4dca7eda592abac994e5073c70be8dbd9b6745  gencode/java/udmi/schema/MappingEvent.java
 d2bf4eea0ca3df47b9ffe31481a52170e2d2bc3a0e7f2eab582e93cc20ccc886  gencode/java/udmi/schema/MappingState.java
 304164da05dc722b6e94cfa68659f0120e2425c94bfb5f5a4c6d796fe13da885  gencode/java/udmi/schema/Metadata.java
 a4e8f69100ab678a8236f481c558d677bbaea3e76c853bbd9262113d2a9c031d  gencode/java/udmi/schema/Metrics.java
@@ -93,14 +94,14 @@ d3968b92497e83a63f18cc0e74484a9807f1bb92db0c92d556ec2caaa143d645  gencode/java/u
 ac6f8fd87c8986cce01e872460c15ff6fe71e3816f9bde610acfe25f7d38c8d4  gencode/java/udmi/schema/ValidationEvent.java
 f7d117dc8b9764acf0c95a13a2bfdfbdf31d1a8ec83a707448aa4d7391ef07e2  gencode/java/udmi/schema/ValidationState.java
 e007ddd1ceeae3603c85110c33e1bb4a418ff9c7a791ca0df25b7ea3caeafd36  gencode/java/udmi/schema/ValidationSummary.java
-f5f6434d1c2ba0bcbff080c6d180b3c410bbf8b25a72b3f670d3fffc2304de43  gencode/python/udmi/schema/__init__.py
+445723ac8a6c5e7072d344fbd865bbefce409bf9a07e194b6d82e9d0751547bd  gencode/python/udmi/schema/__init__.py
 4b25dd95f863059b761269f93adcae7049507924a1c6e74d6856849203c179db  gencode/python/udmi/schema/ancillary_properties.py
-ffca96a15730668a1af22e79800a4f25e689cb801bacce0a3f9c2951dd03653c  gencode/python/udmi/schema/building_config.py
-1049bf13a56024a84c3ab0c64bf96fa65dd63bf64007fda23601e19ec2046d56  gencode/python/udmi/schema/building_config_translation.py
+5ecd6c542f33450cb4ce75d940a6dff4d3bd67d4b9de4aff5ee88abcc301dbff  gencode/python/udmi/schema/building_config.py
+dab4f5fca272ec48c2881bca2b6bc43786ada47fa1f6dd935c35f7ce0eb6b0f6  gencode/python/udmi/schema/building_translation.py
 d4269e665695f7a431108233a427313e609764f421d7657a23bd68d046f11e83  gencode/python/udmi/schema/category.py
 a61b1c5732f01b7efda41a773d5786fad755f371193ce4478b458387ca2a8fe8  gencode/python/udmi/schema/cloud_iot_config.py
 6578d68f65b87b781086e72566de910db4bef365599fe3188862d4d8a81e84fb  gencode/python/udmi/schema/command_discovery.py
-6da1039b08d3b502efa5f483f9e29d7fbdf4482c9b79fa484ff99df01946c6ba  gencode/python/udmi/schema/command_mapping.py
+1254c34d973c9099ae99dcea4534e234e9019f49255e2e27d2afa1bc074fd596  gencode/python/udmi/schema/command_mapping.py
 c5a62f92328e2ede167fc1f53bece6e48696a0ab0e37e3a41f65cd98494ba0d7  gencode/python/udmi/schema/common.py
 b975892df78076dabc797b4c0be87f20b33eacda11f9d1ac1c09be33d4937a87  gencode/python/udmi/schema/config.py
 79eb0299b3751d93c01a5de65eacc717283e99caf0996f3db48a15247f69c8eb  gencode/python/udmi/schema/config_blobset.py
@@ -122,7 +123,7 @@ ba37d2d54df565aba42ef055a0ec961175d180c2e8092a914e62029bc5388857  gencode/python
 82182e3f569ad80dc0751027959c7db9135d10072fbe79f896d63a4cd2f4771f  gencode/python/udmi/schema/event_discovery.py
 ad33b91a7fabb4eed7e49c30a983a2106c96330facbe0f376f94d06e2263d6d0  gencode/python/udmi/schema/event_discovery_family.py
 42e6fb6edda2b6aeae5e5b48ae66f87a5daf336bca1e746b1157e8f1ce31d5d5  gencode/python/udmi/schema/event_discovery_point.py
-24e71ef8c175466cb91612bd2e4428511dbb27030638221ab8cb02c5ee9ea294  gencode/python/udmi/schema/event_mapping.py
+5f41d6252d46f7b63e7a4cac731739b3e672f207eae2c65d448dfefc39b8a2ca  gencode/python/udmi/schema/event_mapping.py
 ddf849bfeb2b87d071cefd5e6feacabc57375a7fff6d72b6d42ffb89f33c859b  gencode/python/udmi/schema/event_pointset.py
 44aff1bc930dbdbadd51ac3fe0e7d9c83ad84a6a9f9d1c809b3fce66cbcd5e00  gencode/python/udmi/schema/event_pointset_point.py
 c3bf9959c821ccc8d0847a2e022e847ac3da8309dc6d35681af3d20148464ee4  gencode/python/udmi/schema/event_system.py

bin/get_logs

@@ -0,0 +1,14 @@
+#!/bin/bash -e
+
+project_id=$1
+
+if [[ -z $project_id ]]; then
+   echo $0 [project_id]
+   false
+fi
+
+for function in udmi_config udmi_target udmi_state udmi_reflect; do
+    echo Capturing $function.log...
+    gcloud --project=$project_id functions logs read $function --sort-by=time_utc --limit=1000 > $function.log
+done
+

common/src/main/java/com/google/udmi/util/SiteModel.java

@@ -9,9 +9,11 @@
 import com.google.common.base.Preconditions;
 import java.io.File;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Map;
 import java.util.Set;
 import java.util.function.BiConsumer;
+import java.util.function.Consumer;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
@@ -36,6 +38,7 @@ public class SiteModel {
 
   final String sitePath;
   private Map<String, Metadata> allMetadata;
+  private Map<String, Device> allDevices;
   private CloudIotConfig cloudIotConfig;
 
   public SiteModel(String sitePath) {
@@ -97,21 +100,25 @@ public EndpointConfiguration makeEndpointConfig(String projectId, String deviceI
     return makeEndpointConfig(projectId, cloudIotConfig, deviceId);
   }
 
-  private Set<String> getAllDevices() {
+  private Set<String> getDeviceIds() {
     Preconditions.checkState(sitePath != null, "sitePath not defined");
     File devicesFile = new File(new File(sitePath), "devices");
     File[] files = Preconditions.checkNotNull(devicesFile.listFiles(), "no files in site devices/");
     return Arrays.stream(files).map(File::getName).collect(Collectors.toSet());
   }
 
   private void loadAllDeviceMetadata() {
-    allMetadata = getAllDevices().stream().collect(toMap(key -> key, this::loadDeviceMetadata));
+    allMetadata = getDeviceIds().stream().collect(toMap(key -> key, this::loadDeviceMetadata));
+    allDevices = getDeviceIds().stream().collect(toMap(key -> key, this::newDevice));
+  }
+
+  private Device newDevice(String deviceId) {
+    return new SiteModel.Device(deviceId);
   }
 
   private Metadata loadDeviceMetadata(String deviceId) {
     Preconditions.checkState(sitePath != null, "sitePath not defined");
-    File devicesFile = new File(new File(sitePath), "devices");
-    File deviceDir = new File(devicesFile, deviceId);
+    File deviceDir = getDeviceDir(deviceId);
     File deviceMetadataFile = new File(deviceDir, "metadata.json");
     try {
       return OBJECT_MAPPER.readValue(deviceMetadataFile, Metadata.class);
@@ -121,11 +128,29 @@ private Metadata loadDeviceMetadata(String deviceId) {
     }
   }
 
+  private File getDeviceDir(String deviceId) {
+    File devicesFile = new File(new File(sitePath), "devices");
+    File deviceDir = new File(devicesFile, deviceId);
+    return deviceDir;
+  }
+
   public Metadata getMetadata(String deviceId) {
     return allMetadata.get(deviceId);
   }
 
-  public void forEachDevice(BiConsumer<String, Metadata> consumer) {
+  public Collection<Device> allDevices() {
+    return allDevices.values();
+  }
+
+  public Collection<String> allDeviceIds() {
+    return allDevices.keySet();
+  }
+
+  public void forEachDevice(Consumer<Device> consumer) {
+    allDevices.values().forEach(consumer);
+  }
+
+  public void forEachMetadata(BiConsumer<String, Metadata> consumer) {
     allMetadata.forEach(consumer);
   }
 
@@ -184,11 +209,28 @@ public String getUpdateTopic() {
     return cloudIotConfig.update_topic;
   }
 
+  public Device getDevice(String deviceId) {
+    return allDevices.get(deviceId);
+  }
+
   public static class ClientInfo {
 
     public String cloudRegion;
     public String projectId;
     public String registryId;
     public String deviceId;
   }
+
+  public class Device {
+
+    public final String deviceId;
+
+    public Device(String deviceId) {
+      this.deviceId = deviceId;
+    }
+
+    public File getFile() {
+      return getDeviceDir(deviceId);
+    }
+  }
 }

dashboard/functions/index.js

@@ -251,7 +251,12 @@ exports.udmi_state = functions.pubsub.topic('udmi_state').onPublish((event) => {
   const msgString = Buffer.from(base64, 'base64').toString();
   const msgObject = JSON.parse(msgString);
 
-  return process_state_update(attributes, msgObject);
+  if (attributes.subFolder) {
+    attributes.subType = STATE_TYPE;
+    return process_state_block(attributes, msgObject);
+  } else {
+    return process_state_update(attributes, msgObject);
+  }
 });
 
 function process_state_update(attributes, msgObject) {
@@ -261,7 +266,7 @@ function process_state_update(attributes, msgObject) {
 
   const commandFolder = `devices/${deviceId}/${STATE_TYPE}/${UPDATE_FOLDER}`;
   promises.push(sendCommand(REFLECT_REGISTRY, registryId, commandFolder, msgObject));
-
+  
   attributes.subFolder = UPDATE_FOLDER;
   attributes.subType = STATE_TYPE;
   promises.push(publishPubsubMessage('udmi_target', attributes, msgObject));
@@ -276,15 +281,22 @@ function process_state_update(attributes, msgObject) {
       attributes.subFolder = block;
       subMsg.timestamp = msgObject.timestamp;
       subMsg.version = msgObject.version;
-      promises.push(publishPubsubMessage('udmi_target', attributes, subMsg));
-      const new_promises = recordMessage(attributes, subMsg);
-      promises.push(...new_promises);
+      promises = promises.concat(process_state_block(attributes, subMsg));
     }
   }
 
   return Promise.all(promises);
 };
 
+function process_state_block(attributes, subMsg) {
+  console.log('Publishing udmi_target', attributes.subType, attributes.subFolder);
+  promises = []
+  promises.push(publishPubsubMessage('udmi_target', attributes, subMsg));
+  const new_promises = recordMessage(attributes, subMsg);
+  promises.push(...new_promises);
+  return promises;
+}
+
 exports.udmi_config = functions.pubsub.topic('udmi_config').onPublish((event) => {
   const attributes = event.attributes;
   const registryId = attributes.deviceRegistryId;

docs/specs/onboarding.md

@@ -55,7 +55,7 @@ sequenceDiagram
 
 1. **[Discovery Config](../../tests/config.tests/discovery.json)** indicates that the _spotter_ should do the needful and scan the local network.
 1. *(Fieldbus Discovery)* scan for fieldbus _device_ information from devices (e.g. BACnet, format out of scope for UDMI):
-  * "I am device `78F936`, with points { `room_temp`, `step_size`, and `operation_count` }, and public key `XYZZYZ`"
+  * "I am device `78F936` with points { `room_temp`, `step_size`, and `operation_count` }"
 2. **[Discovery Events](../../tests/event_discovery.tests/enumeration.json)** wraps the device info from the _spotter_ into a UDMI-normalized format, e.g.:
   * "Device `78F936` has points { }, with a public key `XYZZYZ`"
 3. **[Mapping Config](../../tests/config_mapping.tests/mapping.json)** from the _agent_ indicates that the _engine_ should export responses.