From fe1150adb07ed09b6a8ea7961d4f8c0d5d8eabc5 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Thu, 7 Apr 2022 16:44:59 -0700 Subject: [PATCH 01/27] Examples for discovery --- tests/command_discovery.tests/provision.json | 25 +++++++++++++ tests/config.tests/errors.out | 2 +- tests/config.tests/mapping.json | 23 ++++++++++++ tests/config_pointset.tests/example.json | 25 +++++++++++++ tests/config_pointset.tests/example.out | 0 tests/event_discovery.tests/implicit.json | 2 +- .../event_mapping.tests/building_config.json | 36 +++++++++++++++++++ tests/event_mapping.tests/building_config.out | 0 tests/event_mapping.tests/prediction.json | 13 +++++++ tests/state.tests/mapping.json | 21 +++++++++++ tests/state_pointset.tests/example.json | 36 +++++++++++++++++++ tests/state_pointset.tests/example.out | 0 12 files changed, 181 insertions(+), 2 deletions(-) create mode 100644 tests/command_discovery.tests/provision.json create mode 100644 tests/config.tests/mapping.json create mode 100644 tests/config_pointset.tests/example.json create mode 100644 tests/config_pointset.tests/example.out create mode 100644 tests/event_mapping.tests/building_config.json create mode 100644 tests/event_mapping.tests/building_config.out create mode 100644 tests/event_mapping.tests/prediction.json create mode 100644 tests/state.tests/mapping.json create mode 100644 tests/state_pointset.tests/example.json create mode 100644 tests/state_pointset.tests/example.out diff --git a/tests/command_discovery.tests/provision.json b/tests/command_discovery.tests/provision.json new file mode 100644 index 0000000000..70e20902e5 --- /dev/null +++ b/tests/command_discovery.tests/provision.json @@ -0,0 +1,25 @@ +// +// Information required to provision an on-prem device with a cloud connection +// Relies on protocol-specific mechanism for handling provisioning +// +{ + "version": 1, + "timestamp": "2018-08-26T21:39:29.364Z", + "provision": { + "families": { + "bacnet": { + "92AE93": { + "iot_core": { + "project_id": "udmi-testing", + "cloud_region": "us-central1", + "registry_id": "site-name", + "bridge_hostname": "mqtt.googleapis.com", + "bridge_port": 443, + "device_id": "FCU-123", + "key_algorithm": "RS256" + } + } + } + } + } +} diff --git a/tests/config.tests/errors.out b/tests/config.tests/errors.out index 15974cc6a8..f2e2cd1cca 100644 --- a/tests/config.tests/errors.out +++ b/tests/config.tests/errors.out @@ -1,6 +1,6 @@ 5 schema violations found instance type (string) does not match any allowed primitive type (allowed: ["integer"]) - object instance has properties which are not allowed by the schema: ["config_etag","id","properties","timestamp","version"] + object instance has properties which are not allowed by the schema: ["config_etag","id","properties"] object instance has properties which are not allowed by the schema: ["object_type"] object instance has properties which are not allowed by the schema: ["points","properties","type"] string "2a8b71dwqhdhdddddddddddddddddddddddddddddddddddddddddd8" is too long (length: 55, maximum allowed: 32) diff --git a/tests/config.tests/mapping.json b/tests/config.tests/mapping.json new file mode 100644 index 0000000000..2f739fcdd9 --- /dev/null +++ b/tests/config.tests/mapping.json @@ -0,0 +1,23 @@ +// +// This message represents the entire high-level state of a recommendation model. +// +// Note that building id is included as part of the message envelope. +// +{ + "version": "1.3.14", + "timestamp": "2018-08-28T21:39:29.364Z", + "mapping": { + "devices": { + "FCU-123": { + "applied": "2018-08-27T22:39:12.364Z", // Last time a promotion was successfully applied + "requested": "2018-08-27T22:39:12.364Z", // Last time a model export was requested + "status": { + "message": "Update in progress", + "category": "mapping.apply.start", + "timestamp": "2018-08-28T21:39:30.364Z", + "level": 500 + } + } + } + } +} diff --git a/tests/config_pointset.tests/example.json b/tests/config_pointset.tests/example.json new file mode 100644 index 0000000000..df57209ed4 --- /dev/null +++ b/tests/config_pointset.tests/example.json @@ -0,0 +1,25 @@ +{ + "version": 1, + "timestamp": "2018-08-26T21:39:29.364Z", + "state_etag": "2a8b718", + "set_value_expiry": "2018-08-26T21:49:29.364Z", + "points": { + "room_temperature": { + "set_value": 37.3 + }, + "hallway_temperature": { + "set_value": 37.3 + }, + "hallway_lights": { + "set_value": false + }, + "entryway_lock": { + "set_value": "locked" + }, + "scale_sensor": { + "set_value": 82 + }, + "rocket_motor": { + } + } +} diff --git a/tests/config_pointset.tests/example.out b/tests/config_pointset.tests/example.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/event_discovery.tests/implicit.json b/tests/event_discovery.tests/implicit.json index 8681b5e459..e78775c07e 100644 --- a/tests/event_discovery.tests/implicit.json +++ b/tests/event_discovery.tests/implicit.json @@ -3,7 +3,7 @@ // an implicit object point enumeration. // // Sent on MQTT topic: /devices/{device_id}/events/discovery -// where {device_id} is that of the scanning node (e.g. gateway). The {device_id} +// where {device_id} is that of the scanning node. The {device_id} // of the enumerated node is provided by families.iot.id in the message payload. // { diff --git a/tests/event_mapping.tests/building_config.json b/tests/event_mapping.tests/building_config.json new file mode 100644 index 0000000000..033e7fb939 --- /dev/null +++ b/tests/event_mapping.tests/building_config.json @@ -0,0 +1,36 @@ +// +// This message represents a DBO "buidling_config" entity snippet. +// It could represent data either from an extraction from another system, +// or an output of a promoted staged translation. +// +// The specific building info would be part of the message envelope. +// +{ + "version": "1.3.14", + "timestamp": "2018-08-26T21:39:29.364Z", + "devices": { + "FCU-123": { + "translation": { + "zone_air_temperature_sensor": { + "present_value": "points.temp_1.present_value", + "units": { + "key": "pointset.points.temp_1.units", + "values": { + "degrees_celsius": "degC" + } + } + }, + "supply_air_isolation_damper_command": { + "present_value": "points.damper_1.present_value", + "states": { + "OPEN": "1", + "CLOSED": [ + "2", + "3" + ] + } + } + } + } + } +} diff --git a/tests/event_mapping.tests/building_config.out b/tests/event_mapping.tests/building_config.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/event_mapping.tests/prediction.json b/tests/event_mapping.tests/prediction.json new file mode 100644 index 0000000000..0e2e6ae3fb --- /dev/null +++ b/tests/event_mapping.tests/prediction.json @@ -0,0 +1,13 @@ +// +// This message represents a predictive model for a device. Includes all the +// probabalistic information necessary for an agent (human or otherwise) to +// specify what the actual translation should be. +// +{ + "version": "1.3.14", + "timestamp": "2018-08-26T21:39:29.364Z", + "prediction": { + // Need to include details from staged_equip_response.json + } +} + diff --git a/tests/state.tests/mapping.json b/tests/state.tests/mapping.json new file mode 100644 index 0000000000..276e75a932 --- /dev/null +++ b/tests/state.tests/mapping.json @@ -0,0 +1,21 @@ +// +// This message represents the entire high-level state of a recommendation model. +// Additional messages would be required to detail specific bits (e.g. what was +// discovered or the actual staged or promoted contents). +// +// Note that building id is included as part of the message envelope. +// +{ + "version": "1.3.14", + "timestamp": "2018-08-26T21:39:29.364Z", + "mapping": { + "devices": { + "FCU-123": { + "discovered": "2018-08-26T21:39:29.364Z", // Timestamp of last received discovery message + "staged": "2018-08-28T21:59:18.364Z", // Timestamp of last recommendation staging + "promoted": "2018-08-28T22:39:12.364Z", // Timestamp of last recommendation promotion + "exported": "2018-08-28T22:39:12.364Z", // Timestamp of last time a device mapping was exported + } + } + } +} diff --git a/tests/state_pointset.tests/example.json b/tests/state_pointset.tests/example.json new file mode 100644 index 0000000000..e0a0d45185 --- /dev/null +++ b/tests/state_pointset.tests/example.json @@ -0,0 +1,36 @@ +{ + "version": 1, + "timestamp": "2018-08-26T21:39:29.364Z", + "state_etag": "2a8b718", + "points": { + "room_temperature": { + "value_state": "applied" + }, + "hallway_temperature": { + "value_state": "overridden" + }, + "hallway_lights": { + "value_state": "updating" + }, + "entryway_lock": { + "value_state": "failure", + "status": { + "message": "Failure to confirm point", + "category": "state.pointset.points.config.failure", + "timestamp": "2018-08-26T21:39:28.364Z", + "level": 600 + } + }, + "scale_sensor": { + "value_state": "invalid", + "status": { + "message": "Point is not writable", + "category": "state.pointset.points.config.invalid", + "timestamp": "2018-08-26T21:39:28.364Z", + "level": 600 + } + }, + "rocket_motor": { + } + } +} diff --git a/tests/state_pointset.tests/example.out b/tests/state_pointset.tests/example.out new file mode 100644 index 0000000000..e69de29bb2 From 452ac6dd977b0a92594cd9bfb6dca6a81f394109 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Thu, 7 Apr 2022 16:46:40 -0700 Subject: [PATCH 02/27] Reverting some changes --- tests/config.tests/errors.out | 6 ------ tests/config_pointset.tests/example.out | 0 tests/event_mapping.tests/building_config.out | 0 tests/state_pointset.tests/example.out | 0 4 files changed, 6 deletions(-) delete mode 100644 tests/config.tests/errors.out delete mode 100644 tests/config_pointset.tests/example.out delete mode 100644 tests/event_mapping.tests/building_config.out delete mode 100644 tests/state_pointset.tests/example.out diff --git a/tests/config.tests/errors.out b/tests/config.tests/errors.out deleted file mode 100644 index f2e2cd1cca..0000000000 --- a/tests/config.tests/errors.out +++ /dev/null @@ -1,6 +0,0 @@ -5 schema violations found - instance type (string) does not match any allowed primitive type (allowed: ["integer"]) - object instance has properties which are not allowed by the schema: ["config_etag","id","properties"] - object instance has properties which are not allowed by the schema: ["object_type"] - object instance has properties which are not allowed by the schema: ["points","properties","type"] - string "2a8b71dwqhdhdddddddddddddddddddddddddddddddddddddddddd8" is too long (length: 55, maximum allowed: 32) diff --git a/tests/config_pointset.tests/example.out b/tests/config_pointset.tests/example.out deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/event_mapping.tests/building_config.out b/tests/event_mapping.tests/building_config.out deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/state_pointset.tests/example.out b/tests/state_pointset.tests/example.out deleted file mode 100644 index e69de29bb2..0000000000 From 453373a8edc0870a704b57f45935d6d55c435c31 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Thu, 7 Apr 2022 16:52:45 -0700 Subject: [PATCH 03/27] Cleanup --- tests/config.tests/errors.out | 6 ++++ tests/config_pointset.tests/example.json | 25 ---------------- tests/event_discovery.tests/implicit.json | 2 +- tests/state_pointset.tests/example.json | 36 ----------------------- 4 files changed, 7 insertions(+), 62 deletions(-) create mode 100644 tests/config.tests/errors.out delete mode 100644 tests/config_pointset.tests/example.json delete mode 100644 tests/state_pointset.tests/example.json diff --git a/tests/config.tests/errors.out b/tests/config.tests/errors.out new file mode 100644 index 0000000000..15974cc6a8 --- /dev/null +++ b/tests/config.tests/errors.out @@ -0,0 +1,6 @@ +5 schema violations found + instance type (string) does not match any allowed primitive type (allowed: ["integer"]) + object instance has properties which are not allowed by the schema: ["config_etag","id","properties","timestamp","version"] + object instance has properties which are not allowed by the schema: ["object_type"] + object instance has properties which are not allowed by the schema: ["points","properties","type"] + string "2a8b71dwqhdhdddddddddddddddddddddddddddddddddddddddddd8" is too long (length: 55, maximum allowed: 32) diff --git a/tests/config_pointset.tests/example.json b/tests/config_pointset.tests/example.json deleted file mode 100644 index df57209ed4..0000000000 --- a/tests/config_pointset.tests/example.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "version": 1, - "timestamp": "2018-08-26T21:39:29.364Z", - "state_etag": "2a8b718", - "set_value_expiry": "2018-08-26T21:49:29.364Z", - "points": { - "room_temperature": { - "set_value": 37.3 - }, - "hallway_temperature": { - "set_value": 37.3 - }, - "hallway_lights": { - "set_value": false - }, - "entryway_lock": { - "set_value": "locked" - }, - "scale_sensor": { - "set_value": 82 - }, - "rocket_motor": { - } - } -} diff --git a/tests/event_discovery.tests/implicit.json b/tests/event_discovery.tests/implicit.json index e78775c07e..8681b5e459 100644 --- a/tests/event_discovery.tests/implicit.json +++ b/tests/event_discovery.tests/implicit.json @@ -3,7 +3,7 @@ // an implicit object point enumeration. // // Sent on MQTT topic: /devices/{device_id}/events/discovery -// where {device_id} is that of the scanning node. The {device_id} +// where {device_id} is that of the scanning node (e.g. gateway). The {device_id} // of the enumerated node is provided by families.iot.id in the message payload. // { diff --git a/tests/state_pointset.tests/example.json b/tests/state_pointset.tests/example.json deleted file mode 100644 index e0a0d45185..0000000000 --- a/tests/state_pointset.tests/example.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "version": 1, - "timestamp": "2018-08-26T21:39:29.364Z", - "state_etag": "2a8b718", - "points": { - "room_temperature": { - "value_state": "applied" - }, - "hallway_temperature": { - "value_state": "overridden" - }, - "hallway_lights": { - "value_state": "updating" - }, - "entryway_lock": { - "value_state": "failure", - "status": { - "message": "Failure to confirm point", - "category": "state.pointset.points.config.failure", - "timestamp": "2018-08-26T21:39:28.364Z", - "level": 600 - } - }, - "scale_sensor": { - "value_state": "invalid", - "status": { - "message": "Point is not writable", - "category": "state.pointset.points.config.invalid", - "timestamp": "2018-08-26T21:39:28.364Z", - "level": 600 - } - }, - "rocket_motor": { - } - } -} From 8b06dedf0664319d0988438a8196a6b24c646ec3 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Fri, 8 Apr 2022 08:06:22 -0700 Subject: [PATCH 04/27] Reconciling examples --- tests/event_discovery.json/empty.json | 2 -- tests/event_discovery.json/empty.out | 5 ----- tests/event_discovery.json/errors.json | 22 ------------------- tests/event_discovery.json/errors.out | 9 -------- tests/event_discovery.tests/empty.out | 7 ++++-- tests/event_discovery.tests/errors.json | 4 ++-- tests/event_discovery.tests/errors.out | 15 ++++++++----- .../example.json | 0 .../example.out | 0 9 files changed, 16 insertions(+), 48 deletions(-) delete mode 100644 tests/event_discovery.json/empty.json delete mode 100644 tests/event_discovery.json/empty.out delete mode 100644 tests/event_discovery.json/errors.json delete mode 100644 tests/event_discovery.json/errors.out rename tests/{event_discovery.json => event_discovery.tests}/example.json (100%) rename tests/{event_discovery.json => event_discovery.tests}/example.out (100%) diff --git a/tests/event_discovery.json/empty.json b/tests/event_discovery.json/empty.json deleted file mode 100644 index 2c63c08510..0000000000 --- a/tests/event_discovery.json/empty.json +++ /dev/null @@ -1,2 +0,0 @@ -{ -} diff --git a/tests/event_discovery.json/empty.out b/tests/event_discovery.json/empty.out deleted file mode 100644 index 5a54e24594..0000000000 --- a/tests/event_discovery.json/empty.out +++ /dev/null @@ -1,5 +0,0 @@ -Validating 1 schemas - Validating 1 files against event_discover.json - Against input event_discover.tests/empty.json - 1 schema violations found - object has missing required properties (["families","timestamp","version"]) diff --git a/tests/event_discovery.json/errors.json b/tests/event_discovery.json/errors.json deleted file mode 100644 index acd857e49f..0000000000 --- a/tests/event_discovery.json/errors.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "version": 1, - "timestamp": "2018-08-26T21:39:29.364Z", - "id": "sneakyCASE", - "families": { - "bacnet": { - "id": "92E*A09" - }, - "ip_v4": { - "addr": "192.168.1.2" - }, - "mac-addr": { - "group": "vlan182", - "id": "0e:93:32:11:04:82" - } - }, - "points": { - "bad_entity_name_": { - "present_value": 21.30108642578125 - } - } -} diff --git a/tests/event_discovery.json/errors.out b/tests/event_discovery.json/errors.out deleted file mode 100644 index 413dfd750a..0000000000 --- a/tests/event_discovery.json/errors.out +++ /dev/null @@ -1,9 +0,0 @@ -Validating 1 schemas - Validating 1 files against event_discover.json - Against input event_discover.tests/errors.json - 5 schema violations found - ECMA 262 regex "^[-_.:/0-9a-zA-Z]+$" does not match input string "92E*A09" - object has missing required properties (["id"]) - object instance has properties which are not allowed by the schema: ["addr"] - object instance has properties which are not allowed by the schema: ["id","points"] - object instance has properties which are not allowed by the schema: ["mac-addr"] diff --git a/tests/event_discovery.tests/empty.out b/tests/event_discovery.tests/empty.out index 18e746530f..5a54e24594 100644 --- a/tests/event_discovery.tests/empty.out +++ b/tests/event_discovery.tests/empty.out @@ -1,2 +1,5 @@ -1 schema violations found - object has missing required properties (["generation","timestamp","version"]) +Validating 1 schemas + Validating 1 files against event_discover.json + Against input event_discover.tests/empty.json + 1 schema violations found + object has missing required properties (["families","timestamp","version"]) diff --git a/tests/event_discovery.tests/errors.json b/tests/event_discovery.tests/errors.json index 37fc88cd90..acd857e49f 100644 --- a/tests/event_discovery.tests/errors.json +++ b/tests/event_discovery.tests/errors.json @@ -9,9 +9,9 @@ "ip_v4": { "addr": "192.168.1.2" }, - "mac": { + "mac-addr": { "group": "vlan182", - "id": "0E:93:32:11:04:82" + "id": "0e:93:32:11:04:82" } }, "points": { diff --git a/tests/event_discovery.tests/errors.out b/tests/event_discovery.tests/errors.out index 3582c6aad4..413dfd750a 100644 --- a/tests/event_discovery.tests/errors.out +++ b/tests/event_discovery.tests/errors.out @@ -1,6 +1,9 @@ -5 schema violations found - ECMA 262 regex "^[-_.:0-9A-Z]+$" does not match input string "92E*A09" - object has missing required properties (["generation"]) - object instance has properties which are not allowed by the schema: ["bad_entity_name_"] - object instance has properties which are not allowed by the schema: ["id"] - object instance has properties which are not allowed by the schema: ["ip_v4","mac"] +Validating 1 schemas + Validating 1 files against event_discover.json + Against input event_discover.tests/errors.json + 5 schema violations found + ECMA 262 regex "^[-_.:/0-9a-zA-Z]+$" does not match input string "92E*A09" + object has missing required properties (["id"]) + object instance has properties which are not allowed by the schema: ["addr"] + object instance has properties which are not allowed by the schema: ["id","points"] + object instance has properties which are not allowed by the schema: ["mac-addr"] diff --git a/tests/event_discovery.json/example.json b/tests/event_discovery.tests/example.json similarity index 100% rename from tests/event_discovery.json/example.json rename to tests/event_discovery.tests/example.json diff --git a/tests/event_discovery.json/example.out b/tests/event_discovery.tests/example.out similarity index 100% rename from tests/event_discovery.json/example.out rename to tests/event_discovery.tests/example.out From ad88b0747b4c9383cb7742a79cbe4184039cc536 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Fri, 8 Apr 2022 08:12:53 -0700 Subject: [PATCH 05/27] Docs tweaks --- tests/config.tests/mapping.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/config.tests/mapping.json b/tests/config.tests/mapping.json index 2f739fcdd9..03b9051526 100644 --- a/tests/config.tests/mapping.json +++ b/tests/config.tests/mapping.json @@ -1,7 +1,7 @@ // -// This message represents the entire high-level state of a recommendation model. +// This message represents the entire high-level state of a mapping recommendation model. // -// Note that building id is included as part of the message envelope. +// The building id is included as part of the message envelope. // { "version": "1.3.14", From a822c5f0d1fd5bc3bb42bc8d64f8278336aec840 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Fri, 8 Apr 2022 09:16:17 -0700 Subject: [PATCH 06/27] Updating docs --- tests/config.tests/mapping.json | 12 +++++++++--- tests/state.tests/mapping.json | 16 +++++++++++----- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/tests/config.tests/mapping.json b/tests/config.tests/mapping.json index 03b9051526..572a3ed9fd 100644 --- a/tests/config.tests/mapping.json +++ b/tests/config.tests/mapping.json @@ -1,16 +1,22 @@ // -// This message represents the entire high-level state of a mapping recommendation model. +// This message represents the entire high-level config of a mapping recommendation model. +// This is essentially an "input" from the outside world that might be relevant to the mapper internals. // // The building id is included as part of the message envelope. // +// Basic comparisons are with values in the related state message.json block, which can be handled as logical +// timestamp comparisons: +// applied < promoted: This means a entry has been promoted but has not yet been applied by the external system +// requested > exported: This means that an export has been requested but not yet triggered (mapper should export data) +// { "version": "1.3.14", "timestamp": "2018-08-28T21:39:29.364Z", "mapping": { "devices": { "FCU-123": { - "applied": "2018-08-27T22:39:12.364Z", // Last time a promotion was successfully applied - "requested": "2018-08-27T22:39:12.364Z", // Last time a model export was requested + "applied": "2018-08-27T22:39:12.364Z", // Last time a promotion was successfully applied externally + "requested": "2018-08-27T22:39:12.364Z", // Timestamp of requested model export "status": { "message": "Update in progress", "category": "mapping.apply.start", diff --git a/tests/state.tests/mapping.json b/tests/state.tests/mapping.json index 276e75a932..d237352c8a 100644 --- a/tests/state.tests/mapping.json +++ b/tests/state.tests/mapping.json @@ -1,20 +1,26 @@ // -// This message represents the entire high-level state of a recommendation model. +// This message represents the entire high-level state of a mapping recommendation. // Additional messages would be required to detail specific bits (e.g. what was // discovered or the actual staged or promoted contents). // // Note that building id is included as part of the message envelope. // +// Gendral gendral workflow is: +// 1. Mapper receives discovery message +// 2. Internal algorithm predicts what it might be and gives options +// 3. Some process for promoting a single option based on prediction +// 4. Result of promotion are exported for external consumption +// { "version": "1.3.14", "timestamp": "2018-08-26T21:39:29.364Z", "mapping": { "devices": { "FCU-123": { - "discovered": "2018-08-26T21:39:29.364Z", // Timestamp of last received discovery message - "staged": "2018-08-28T21:59:18.364Z", // Timestamp of last recommendation staging - "promoted": "2018-08-28T22:39:12.364Z", // Timestamp of last recommendation promotion - "exported": "2018-08-28T22:39:12.364Z", // Timestamp of last time a device mapping was exported + "discovered": "2018-08-26T21:39:29.364Z", // Last received discovery message + "predicted": "2018-08-28T21:59:18.364Z", // Last recommendation staging (result of automatic prediction) + "promoted": "2018-08-28T22:39:12.364Z", // Last recommendation promotion (result of manual QA) + "exported": "2018-08-28T22:39:12.364Z", // Last time this device mapping was exported } } } From 3beb3edad51509d686c3383f8c00aaf9296d52cd Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Fri, 8 Apr 2022 09:17:28 -0700 Subject: [PATCH 07/27] REmoving provisioning example --- tests/command_discovery.tests/provision.json | 25 -------------------- 1 file changed, 25 deletions(-) delete mode 100644 tests/command_discovery.tests/provision.json diff --git a/tests/command_discovery.tests/provision.json b/tests/command_discovery.tests/provision.json deleted file mode 100644 index 70e20902e5..0000000000 --- a/tests/command_discovery.tests/provision.json +++ /dev/null @@ -1,25 +0,0 @@ -// -// Information required to provision an on-prem device with a cloud connection -// Relies on protocol-specific mechanism for handling provisioning -// -{ - "version": 1, - "timestamp": "2018-08-26T21:39:29.364Z", - "provision": { - "families": { - "bacnet": { - "92AE93": { - "iot_core": { - "project_id": "udmi-testing", - "cloud_region": "us-central1", - "registry_id": "site-name", - "bridge_hostname": "mqtt.googleapis.com", - "bridge_port": 443, - "device_id": "FCU-123", - "key_algorithm": "RS256" - } - } - } - } - } -} From 0e3c0e0bf7d71109024bb4a4a5dedf5f6238d811 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Fri, 8 Apr 2022 13:20:38 -0700 Subject: [PATCH 08/27] Adding mapping_format --- tests/config.tests/mapping.json | 1 + tests/event_mapping.tests/building_config.json | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/config.tests/mapping.json b/tests/config.tests/mapping.json index 572a3ed9fd..7be2b66a58 100644 --- a/tests/config.tests/mapping.json +++ b/tests/config.tests/mapping.json @@ -12,6 +12,7 @@ { "version": "1.3.14", "timestamp": "2018-08-28T21:39:29.364Z", + "mapping_format": "building_config", "mapping": { "devices": { "FCU-123": { diff --git a/tests/event_mapping.tests/building_config.json b/tests/event_mapping.tests/building_config.json index 033e7fb939..e7a354e039 100644 --- a/tests/event_mapping.tests/building_config.json +++ b/tests/event_mapping.tests/building_config.json @@ -8,6 +8,7 @@ { "version": "1.3.14", "timestamp": "2018-08-26T21:39:29.364Z", + "mapping_format": "building_config", "devices": { "FCU-123": { "translation": { From 9dce1fa6ade7fcfbb3442ce696838b4b5b2def18 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Sat, 9 Apr 2022 09:20:51 -0700 Subject: [PATCH 09/27] Adding guid --- .../building_config.json | 38 +++++++++++++++++++ tests/config.tests/mapping.json | 1 + .../event_mapping.tests/building_config.json | 1 + tests/state.tests/mapping.json | 1 + 4 files changed, 41 insertions(+) create mode 100644 tests/command_mapping.tests/building_config.json diff --git a/tests/command_mapping.tests/building_config.json b/tests/command_mapping.tests/building_config.json new file mode 100644 index 0000000000..b94432dd8e --- /dev/null +++ b/tests/command_mapping.tests/building_config.json @@ -0,0 +1,38 @@ +// +// This message represents a DBO "buidling_config" entity snippet. +// It could represent data either from an extraction from another system, +// or an output of a promoted staged translation. +// +// The specific building info would be part of the message envelope. +// +{ + "version": "1.3.14", + "timestamp": "2018-08-26T21:39:29.364Z", + "mapping_format": "building_config", + "devices": { + "FCU-123": { + "guid": "21387BBA787ADD", + "translation": { + "zone_air_temperature_sensor": { + "present_value": "points.temp_1.present_value", + "units": { + "key": "pointset.points.temp_1.units", + "values": { + "degrees_celsius": "degC" + } + } + }, + "supply_air_isolation_damper_command": { + "present_value": "points.damper_1.present_value", + "states": { + "OPEN": "1", + "CLOSED": [ + "2", + "3" + ] + } + } + } + } + } +} diff --git a/tests/config.tests/mapping.json b/tests/config.tests/mapping.json index 7be2b66a58..4f380b716f 100644 --- a/tests/config.tests/mapping.json +++ b/tests/config.tests/mapping.json @@ -16,6 +16,7 @@ "mapping": { "devices": { "FCU-123": { + "guid": "21387BBA787ADD", "applied": "2018-08-27T22:39:12.364Z", // Last time a promotion was successfully applied externally "requested": "2018-08-27T22:39:12.364Z", // Timestamp of requested model export "status": { diff --git a/tests/event_mapping.tests/building_config.json b/tests/event_mapping.tests/building_config.json index e7a354e039..b94432dd8e 100644 --- a/tests/event_mapping.tests/building_config.json +++ b/tests/event_mapping.tests/building_config.json @@ -11,6 +11,7 @@ "mapping_format": "building_config", "devices": { "FCU-123": { + "guid": "21387BBA787ADD", "translation": { "zone_air_temperature_sensor": { "present_value": "points.temp_1.present_value", diff --git a/tests/state.tests/mapping.json b/tests/state.tests/mapping.json index d237352c8a..23857e8af0 100644 --- a/tests/state.tests/mapping.json +++ b/tests/state.tests/mapping.json @@ -17,6 +17,7 @@ "mapping": { "devices": { "FCU-123": { + "guid": "21387BBA787ADD", "discovered": "2018-08-26T21:39:29.364Z", // Last received discovery message "predicted": "2018-08-28T21:59:18.364Z", // Last recommendation staging (result of automatic prediction) "promoted": "2018-08-28T22:39:12.364Z", // Last recommendation promotion (result of manual QA) From 35aa1646f38ff37c7dc7c5ca30e622b98400b450 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Mon, 11 Apr 2022 08:21:13 -0700 Subject: [PATCH 10/27] Updating examples --- .../building_config.json | 47 ++++++++----------- tests/config.tests/mapping.json | 16 +++---- .../event_mapping.tests/building_config.json | 43 ++++++++--------- tests/event_mapping.tests/prediction.json | 4 +- tests/state.tests/mapping.json | 18 +++---- 5 files changed, 58 insertions(+), 70 deletions(-) diff --git a/tests/command_mapping.tests/building_config.json b/tests/command_mapping.tests/building_config.json index b94432dd8e..56fbbad8ab 100644 --- a/tests/command_mapping.tests/building_config.json +++ b/tests/command_mapping.tests/building_config.json @@ -1,38 +1,31 @@ // -// This message represents a DBO "buidling_config" entity snippet. -// It could represent data either from an extraction from another system, -// or an output of a promoted staged translation. +// This message represents an import of an existing device mapping. // -// The specific building info would be part of the message envelope. +// Note that the building and device id are included as part of the message envelope. // { "version": "1.3.14", "timestamp": "2018-08-26T21:39:29.364Z", - "mapping_format": "building_config", - "devices": { - "FCU-123": { - "guid": "21387BBA787ADD", - "translation": { - "zone_air_temperature_sensor": { - "present_value": "points.temp_1.present_value", - "units": { - "key": "pointset.points.temp_1.units", - "values": { - "degrees_celsius": "degC" - } - } - }, - "supply_air_isolation_damper_command": { - "present_value": "points.damper_1.present_value", - "states": { - "OPEN": "1", - "CLOSED": [ - "2", - "3" - ] - } + "guid": "21387BBA787ADD", + "translation": { + "zone_air_temperature_sensor": { + "present_value": "points.temp_1.present_value", + "units": { + "key": "pointset.points.temp_1.units", + "values": { + "degrees_celsius": "degC" } } + }, + "supply_air_isolation_damper_command": { + "present_value": "points.damper_1.present_value", + "states": { + "OPEN": "1", + "CLOSED": [ + "2", + "3" + ] + } } } } diff --git a/tests/config.tests/mapping.json b/tests/config.tests/mapping.json index 4f380b716f..0f6742ac3f 100644 --- a/tests/config.tests/mapping.json +++ b/tests/config.tests/mapping.json @@ -2,23 +2,21 @@ // This message represents the entire high-level config of a mapping recommendation model. // This is essentially an "input" from the outside world that might be relevant to the mapper internals. // -// The building id is included as part of the message envelope. +// The building id is defined by the message envelope. // -// Basic comparisons are with values in the related state message.json block, which can be handled as logical -// timestamp comparisons: -// applied < promoted: This means a entry has been promoted but has not yet been applied by the external system -// requested > exported: This means that an export has been requested but not yet triggered (mapper should export data) +// Basic comparisons are with values in the related state message block as a logical timestamp comparisons: +// applied < promoted: This means a entry has been promoted but has not yet been applied by the external system +// requested > exported: This means that an export has been requested but not yet triggered (mapper should export data) // { "version": "1.3.14", "timestamp": "2018-08-28T21:39:29.364Z", - "mapping_format": "building_config", "mapping": { "devices": { "FCU-123": { - "guid": "21387BBA787ADD", - "applied": "2018-08-27T22:39:12.364Z", // Last time a promotion was successfully applied externally - "requested": "2018-08-27T22:39:12.364Z", // Timestamp of requested model export + "guid": "21387BBA787ADD", + "applied": "2018-08-27T22:39:12.364Z", // Last time a promotion was successfully applied externally + "requested": "2018-08-27T22:39:12.364Z", // Timestamp of requested model export "status": { "message": "Update in progress", "category": "mapping.apply.start", diff --git a/tests/event_mapping.tests/building_config.json b/tests/event_mapping.tests/building_config.json index b94432dd8e..a37a82bc1d 100644 --- a/tests/event_mapping.tests/building_config.json +++ b/tests/event_mapping.tests/building_config.json @@ -3,36 +3,31 @@ // It could represent data either from an extraction from another system, // or an output of a promoted staged translation. // -// The specific building info would be part of the message envelope. +// The specific building and device id would be part of the message envelope. // { "version": "1.3.14", "timestamp": "2018-08-26T21:39:29.364Z", - "mapping_format": "building_config", - "devices": { - "FCU-123": { - "guid": "21387BBA787ADD", - "translation": { - "zone_air_temperature_sensor": { - "present_value": "points.temp_1.present_value", - "units": { - "key": "pointset.points.temp_1.units", - "values": { - "degrees_celsius": "degC" - } - } - }, - "supply_air_isolation_damper_command": { - "present_value": "points.damper_1.present_value", - "states": { - "OPEN": "1", - "CLOSED": [ - "2", - "3" - ] - } + "guid": "21387BBA787ADD", + "translation": { + "zone_air_temperature_sensor": { + "present_value": "points.temp_1.present_value", + "units": { + "key": "pointset.points.temp_1.units", + "values": { + "degrees_celsius": "degC" } } + }, + "supply_air_isolation_damper_command": { + "present_value": "points.damper_1.present_value", + "states": { + "OPEN": "1", + "CLOSED": [ + "2", + "3" + ] + } } } } diff --git a/tests/event_mapping.tests/prediction.json b/tests/event_mapping.tests/prediction.json index 0e2e6ae3fb..05bf20f109 100644 --- a/tests/event_mapping.tests/prediction.json +++ b/tests/event_mapping.tests/prediction.json @@ -1,7 +1,8 @@ // // This message represents a predictive model for a device. Includes all the // probabalistic information necessary for an agent (human or otherwise) to -// specify what the actual translation should be. +// specify what the actual translation should be. This is an intermediate +// state that requires 'promotion' before being properly exported. // { "version": "1.3.14", @@ -10,4 +11,3 @@ // Need to include details from staged_equip_response.json } } - diff --git a/tests/state.tests/mapping.json b/tests/state.tests/mapping.json index 23857e8af0..e31b62b68d 100644 --- a/tests/state.tests/mapping.json +++ b/tests/state.tests/mapping.json @@ -1,15 +1,16 @@ // // This message represents the entire high-level state of a mapping recommendation. -// Additional messages would be required to detail specific bits (e.g. what was -// discovered or the actual staged or promoted contents). +// Specific details (i.e., the actual recommendations) are enapsulated in a separate +// message. // -// Note that building id is included as part of the message envelope. +// Note that building id is defined as part of the message envelope. // // Gendral gendral workflow is: -// 1. Mapper receives discovery message -// 2. Internal algorithm predicts what it might be and gives options -// 3. Some process for promoting a single option based on prediction -// 4. Result of promotion are exported for external consumption +// 1. Import: Existing mappings can be imported from an external source +// 2. Discovery: Mapper receives discovery message +// 3. Prediction: Internal algorithm predicts what it might be and gives options +// 4. Promotion: Some process for promoting a single option based on prediction +// 5. Export: Result of promotion are exported for external consumption // { "version": "1.3.14", @@ -17,7 +18,8 @@ "mapping": { "devices": { "FCU-123": { - "guid": "21387BBA787ADD", + "guid": "21387BBA787ADD", + "imported": "2018-08-26T21:39:28.364Z", // Last time the mapping was imported "discovered": "2018-08-26T21:39:29.364Z", // Last received discovery message "predicted": "2018-08-28T21:59:18.364Z", // Last recommendation staging (result of automatic prediction) "promoted": "2018-08-28T22:39:12.364Z", // Last recommendation promotion (result of manual QA) From 7d58c8e5501fd52e64405af3f2c93e064c29dc74 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Mon, 11 Apr 2022 11:11:25 -0700 Subject: [PATCH 11/27] Adding staged eqip.txt --- tests/event_mapping.tests/prediction.json | 2 +- .../staged_equip_response.txt | 51 +++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 tests/event_mapping.tests/staged_equip_response.txt diff --git a/tests/event_mapping.tests/prediction.json b/tests/event_mapping.tests/prediction.json index 05bf20f109..5ed99f1f28 100644 --- a/tests/event_mapping.tests/prediction.json +++ b/tests/event_mapping.tests/prediction.json @@ -8,6 +8,6 @@ "version": "1.3.14", "timestamp": "2018-08-26T21:39:29.364Z", "prediction": { - // Need to include details from staged_equip_response.json + // TODO: Include details from staged_equip_response.txt } } diff --git a/tests/event_mapping.tests/staged_equip_response.txt b/tests/event_mapping.tests/staged_equip_response.txt new file mode 100644 index 0000000000..867b566f28 --- /dev/null +++ b/tests/event_mapping.tests/staged_equip_response.txt @@ -0,0 +1,51 @@ +{ + "building_name": "US-MTV-1804", // building id from discovery message + "equipment": + [ + { + "data": + { + "cloud_id": "270074000670001", // from UDMI discovery + "model.equip_type_predictions": // top five predictions + { + "foo_1": 0.10, + "foo_2": 0.15, + "foo_3": 0.20, + "foo_4": 0.25, + "foo_5": 0.30 + }, + "equip_type": "foo_5", // top prediction + "model.equip_id": "ENTITY-GUID-1", // predicted equipment instance ID + "model.parent": null, // predicted parent equipment ID + }, + "created": "2022-03-25T18:16:36+00:00", + "last_promoted": null, + "modified": null, + } + ], + "points_by_equip_id": + { + "ENTITY-GUID-1": + [ + { + "data": + { + "object_identifier": "('analogInupt', 1)", // from UDMI discovery + "model.point_type_predictions": // top five predictions + { + "bar_1": 0.10, + "bar_2": 0.15, + "bar_3": 0.20, + "bar_4": 0.25, + "bar_5": 0.30 + }, + "point_type": "bar_5", // top prediction + }, + "created": "2022-03-25T18:16:36+00:00", + "last_promoted": null, + "modified": null, + "equip_id": "ENTITY-GUID-1", + } + ] + } +} From 400d7fd9c080ffd64c91a86b8b83478dacc54026 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Tue, 12 Apr 2022 07:22:03 -0700 Subject: [PATCH 12/27] Updating staged_equp_repsonse --- .../staged_equip_response.txt | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/tests/event_mapping.tests/staged_equip_response.txt b/tests/event_mapping.tests/staged_equip_response.txt index 867b566f28..d7e69fbb21 100644 --- a/tests/event_mapping.tests/staged_equip_response.txt +++ b/tests/event_mapping.tests/staged_equip_response.txt @@ -5,27 +5,28 @@ { "data": { - "cloud_id": "270074000670001", // from UDMI discovery + "device_id": "4b:9a:84:17:17:6c", // from UDMI discovery, ties back to a specific device "model.equip_type_predictions": // top five predictions { - "foo_1": 0.10, - "foo_2": 0.15, - "foo_3": 0.20, - "foo_4": 0.25, - "foo_5": 0.30 + "AHU": 0.10, + "BLR": 0.15, + "HX": 0.20, + "MAU": 0.25, + "VAV": 0.30 }, - "equip_type": "foo_5", // top prediction - "model.equip_id": "ENTITY-GUID-1", // predicted equipment instance ID - "model.parent": null, // predicted parent equipment ID + "entity_type": "VAV", // top prediction + "model.equip_name": "VAV-1", // predicted equipment instance ID + "guid": "8cf8c6ea-c8aa-412c-914d-947947fa51ba", // UUID V4 + "model.parent": "AHU-1", // predicted parent equipment ID }, "created": "2022-03-25T18:16:36+00:00", "last_promoted": null, "modified": null, } ], - "points_by_equip_id": + "points_by_equip_name": { - "ENTITY-GUID-1": + "VAV-1": [ { "data": @@ -33,18 +34,18 @@ "object_identifier": "('analogInupt', 1)", // from UDMI discovery "model.point_type_predictions": // top five predictions { - "bar_1": 0.10, - "bar_2": 0.15, - "bar_3": 0.20, - "bar_4": 0.25, - "bar_5": 0.30 + "building_air_static_pressure_sensor": 0.10, + "condensing_supply_water_temperature_sensor": 0.15, + "exhaust_fan_power_sensor": 0.20, + "discharge_fan_speed_percentage_sensor": 0.25, + "zone_air_relative_humidity_sensor": 0.30 }, - "point_type": "bar_5", // top prediction + "telemetry_field_name": "zone_air_relative_humidity_sensor", // top prediction }, "created": "2022-03-25T18:16:36+00:00", "last_promoted": null, "modified": null, - "equip_id": "ENTITY-GUID-1", + "entity.guid": "8cf8c6ea-c8aa-412c-914d-947947fa51ba", } ] } From ba037a810b6675269ec5ad3e589d41dfeb9e663f Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Fri, 17 Jun 2022 08:47:20 -0700 Subject: [PATCH 13/27] Adjusting schema locations --- .../mapping.json | 22 +++++++++---------- .../mapping.json | 18 +++++++-------- 2 files changed, 18 insertions(+), 22 deletions(-) rename tests/{config.tests => config_mapping.tests}/mapping.json (59%) rename tests/{state.tests => state_mapping.tests}/mapping.json (55%) diff --git a/tests/config.tests/mapping.json b/tests/config_mapping.tests/mapping.json similarity index 59% rename from tests/config.tests/mapping.json rename to tests/config_mapping.tests/mapping.json index 0f6742ac3f..e89d3cfd53 100644 --- a/tests/config.tests/mapping.json +++ b/tests/config_mapping.tests/mapping.json @@ -11,18 +11,16 @@ { "version": "1.3.14", "timestamp": "2018-08-28T21:39:29.364Z", - "mapping": { - "devices": { - "FCU-123": { - "guid": "21387BBA787ADD", - "applied": "2018-08-27T22:39:12.364Z", // Last time a promotion was successfully applied externally - "requested": "2018-08-27T22:39:12.364Z", // Timestamp of requested model export - "status": { - "message": "Update in progress", - "category": "mapping.apply.start", - "timestamp": "2018-08-28T21:39:30.364Z", - "level": 500 - } + "devices": { + "FCU-123": { + "guid": "21387BBA787ADD", + "applied": "2018-08-27T22:39:12.364Z", // Last time a promotion was successfully applied externally + "requested": "2018-08-27T22:39:12.364Z", // Timestamp of requested model export + "status": { + "message": "Update in progress", + "category": "mapping.apply.start", + "timestamp": "2018-08-28T21:39:30.364Z", + "level": 500 } } } diff --git a/tests/state.tests/mapping.json b/tests/state_mapping.tests/mapping.json similarity index 55% rename from tests/state.tests/mapping.json rename to tests/state_mapping.tests/mapping.json index e31b62b68d..09c8c3e74f 100644 --- a/tests/state.tests/mapping.json +++ b/tests/state_mapping.tests/mapping.json @@ -15,16 +15,14 @@ { "version": "1.3.14", "timestamp": "2018-08-26T21:39:29.364Z", - "mapping": { - "devices": { - "FCU-123": { - "guid": "21387BBA787ADD", - "imported": "2018-08-26T21:39:28.364Z", // Last time the mapping was imported - "discovered": "2018-08-26T21:39:29.364Z", // Last received discovery message - "predicted": "2018-08-28T21:59:18.364Z", // Last recommendation staging (result of automatic prediction) - "promoted": "2018-08-28T22:39:12.364Z", // Last recommendation promotion (result of manual QA) - "exported": "2018-08-28T22:39:12.364Z", // Last time this device mapping was exported - } + "devices": { + "FCU-123": { + "guid": "21387BBA787ADD", + "imported": "2018-08-26T21:39:28.364Z", // Last time the mapping was imported + "discovered": "2018-08-26T21:39:29.364Z", // Last received discovery message + "predicted": "2018-08-28T21:59:18.364Z", // Last recommendation staging (result of automatic prediction) + "promoted": "2018-08-28T22:39:12.364Z", // Last recommendation promotion (result of manual QA) + "exported": "2018-08-28T22:39:12.364Z", // Last time this device mapping was exported } } } From a3ed46bfce3736900dfe08301ccf1604b00540cb Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Fri, 17 Jun 2022 09:07:13 -0700 Subject: [PATCH 14/27] Update and rename --- .../{building_config.json => mapping.json} | 0 tests/config_mapping.tests/mapping.json | 16 ++++++++++------ .../{building_config.json => mapping.json} | 5 ++--- tests/state_mapping.tests/mapping.json | 11 +++++++---- 4 files changed, 19 insertions(+), 13 deletions(-) rename tests/command_mapping.tests/{building_config.json => mapping.json} (100%) rename tests/event_mapping.tests/{building_config.json => mapping.json} (77%) diff --git a/tests/command_mapping.tests/building_config.json b/tests/command_mapping.tests/mapping.json similarity index 100% rename from tests/command_mapping.tests/building_config.json rename to tests/command_mapping.tests/mapping.json diff --git a/tests/config_mapping.tests/mapping.json b/tests/config_mapping.tests/mapping.json index e89d3cfd53..d71e463563 100644 --- a/tests/config_mapping.tests/mapping.json +++ b/tests/config_mapping.tests/mapping.json @@ -1,12 +1,16 @@ // -// This message represents the entire high-level config of a mapping recommendation model. -// This is essentially an "input" from the outside world that might be relevant to the mapper internals. +// This message represents the entire high-level config of a recommendation model, +// as given to the mapping component. It therefore reflects the external (to mapper) +// understanding of everything. // // The building id is defined by the message envelope. // -// Basic comparisons are with values in the related state message block as a logical timestamp comparisons: -// applied < promoted: This means a entry has been promoted but has not yet been applied by the external system -// requested > exported: This means that an export has been requested but not yet triggered (mapper should export data) +// General overview of the external representation: +// 1. Applied: An exported recommendation was received and applied to the system. +// 2. Request: Last time an export was requested from the system. +// +// Comaprison of timestamps would trigger action by the external (non-mapper) system: +// imported < applied: The system should import (send command) the applied mapping into the mapper // { "version": "1.3.14", @@ -14,7 +18,7 @@ "devices": { "FCU-123": { "guid": "21387BBA787ADD", - "applied": "2018-08-27T22:39:12.364Z", // Last time a promotion was successfully applied externally + "applied": "2018-08-27T22:39:12.364Z", // Last time a mapping was successfully applied externally "requested": "2018-08-27T22:39:12.364Z", // Timestamp of requested model export "status": { "message": "Update in progress", diff --git a/tests/event_mapping.tests/building_config.json b/tests/event_mapping.tests/mapping.json similarity index 77% rename from tests/event_mapping.tests/building_config.json rename to tests/event_mapping.tests/mapping.json index a37a82bc1d..d8da42a2c8 100644 --- a/tests/event_mapping.tests/building_config.json +++ b/tests/event_mapping.tests/mapping.json @@ -1,7 +1,6 @@ // -// This message represents a DBO "buidling_config" entity snippet. -// It could represent data either from an extraction from another system, -// or an output of a promoted staged translation. +// This message represents an output from the mapper expressing the mapping +// for a particular device. // // The specific building and device id would be part of the message envelope. // diff --git a/tests/state_mapping.tests/mapping.json b/tests/state_mapping.tests/mapping.json index 09c8c3e74f..a4f3e54915 100644 --- a/tests/state_mapping.tests/mapping.json +++ b/tests/state_mapping.tests/mapping.json @@ -1,17 +1,20 @@ // -// This message represents the entire high-level state of a mapping recommendation. -// Specific details (i.e., the actual recommendations) are enapsulated in a separate -// message. +// This message represents the entire high-level state of a recommendation, +// as given by the mapping component. It therefore reflects the internal state +// of the mapper as might be of interest to the outside world. // // Note that building id is defined as part of the message envelope. // -// Gendral gendral workflow is: +// General workflow is (not all steps will always be present): // 1. Import: Existing mappings can be imported from an external source // 2. Discovery: Mapper receives discovery message // 3. Prediction: Internal algorithm predicts what it might be and gives options // 4. Promotion: Some process for promoting a single option based on prediction // 5. Export: Result of promotion are exported for external consumption // +// Comparison of timestamps with the related config block would trigger action by the mapper: +// requested > exported: The mapper should export (send event) the indicated mapping +// { "version": "1.3.14", "timestamp": "2018-08-26T21:39:29.364Z", From 2940a2eea155d767b483147cf4c2f66fafc5dbc4 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Fri, 17 Jun 2022 09:10:44 -0700 Subject: [PATCH 15/27] Docs tweaks --- tests/command_mapping.tests/mapping.json | 2 +- tests/config_mapping.tests/mapping.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/command_mapping.tests/mapping.json b/tests/command_mapping.tests/mapping.json index 56fbbad8ab..97a26464e7 100644 --- a/tests/command_mapping.tests/mapping.json +++ b/tests/command_mapping.tests/mapping.json @@ -1,5 +1,5 @@ // -// This message represents an import of an existing device mapping. +// This message represents an import of an existing device mapping into the mapper. // // Note that the building and device id are included as part of the message envelope. // diff --git a/tests/config_mapping.tests/mapping.json b/tests/config_mapping.tests/mapping.json index d71e463563..d3b25ae1b1 100644 --- a/tests/config_mapping.tests/mapping.json +++ b/tests/config_mapping.tests/mapping.json @@ -1,7 +1,7 @@ // // This message represents the entire high-level config of a recommendation model, // as given to the mapping component. It therefore reflects the external (to mapper) -// understanding of everything. +// view of what has happened outside of the mapper itself. // // The building id is defined by the message envelope. // From 782237584ece5c1517b12373f7e6f778ff411cf4 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Sun, 7 Aug 2022 07:36:53 -0700 Subject: [PATCH 16/27] Force reset of udmif --- .../src/device/dao/mongodb/MongoDeviceDAO.ts | 2 +- .../src/device/dao/static/StaticDeviceDAO.ts | 1 + udmif/event-handler/src/DeviceDao.ts | 4 ++-- .../src/DeviceDocumentFactory.ts | 18 +++++++++++++---- udmif/event-handler/src/DocumentTypeUtil.ts | 2 ++ udmif/event-handler/src/UdmiMessageHandler.ts | 2 +- .../src/__tests__/DeviceDao.spec.ts | 5 +++-- .../src/__tests__/DeviceDaoFactory.spec.ts | 1 + .../src/__tests__/DeviceDocument.spec.ts | 2 +- .../__tests__/DeviceDocumentFactory.spec.ts | 20 ++++++++++++++++--- .../src/__tests__/DeviceTypeUtils.spec.ts | 2 ++ .../src/__tests__/Points.spec.ts | 9 ++------- udmif/web/src/app/app.module.ts | 2 +- udmif/web/src/app/auth/auth.module.ts | 2 +- udmif/web/src/app/auth/auth.service.ts | 4 ++-- udmif/web/src/app/graphql/graphql.module.ts | 2 +- 16 files changed, 52 insertions(+), 26 deletions(-) diff --git a/udmif/api/src/device/dao/mongodb/MongoDeviceDAO.ts b/udmif/api/src/device/dao/mongodb/MongoDeviceDAO.ts index cdf5f0bde7..88604238d6 100644 --- a/udmif/api/src/device/dao/mongodb/MongoDeviceDAO.ts +++ b/udmif/api/src/device/dao/mongodb/MongoDeviceDAO.ts @@ -18,7 +18,7 @@ import { getAggregate } from './MongoAggregateBuilder'; // this class exists to return sorted, and filtered data from MongoDB export class MongoDeviceDAO implements DeviceDAO { - constructor(private db: Db) {} + constructor(private db: Db) { } async getDevices(searchOptions: SearchOptions): Promise { return this.db diff --git a/udmif/api/src/device/dao/static/StaticDeviceDAO.ts b/udmif/api/src/device/dao/static/StaticDeviceDAO.ts index e899bc69a0..fa86bd5455 100644 --- a/udmif/api/src/device/dao/static/StaticDeviceDAO.ts +++ b/udmif/api/src/device/dao/static/StaticDeviceDAO.ts @@ -50,6 +50,7 @@ export class StaticDeviceDAO implements DeviceDAO { } public async getFilteredDeviceCount(searchOptions: SearchOptions): Promise { + let filteredDevices = this.devices; if (searchOptions.filter) { const filters: Filter[] = fromString(searchOptions.filter); diff --git a/udmif/event-handler/src/DeviceDao.ts b/udmif/event-handler/src/DeviceDao.ts index 43dec976ad..92e8c76f87 100644 --- a/udmif/event-handler/src/DeviceDao.ts +++ b/udmif/event-handler/src/DeviceDao.ts @@ -10,12 +10,12 @@ export interface DeviceDao { } export class DefaultDeviceDao implements DeviceDao { - constructor(private collection: Collection) {} + constructor(private collection: Collection) { } /** * Updates a device document if it is found using the device key, else it will insert a new device document * @param {DeviceKey} deviceKey - * @param {DeviceDocument} deviceDocument + * @param {DeviceDocument} deviceDocument */ async upsert(deviceKey: DeviceKey, deviceDocument: DeviceDocument): Promise { // we're using upsert which will allow document updates if it already exists and a document cretion if it does not diff --git a/udmif/event-handler/src/DeviceDocumentFactory.ts b/udmif/event-handler/src/DeviceDocumentFactory.ts index 4b85f88b8f..3eb4901505 100644 --- a/udmif/event-handler/src/DeviceDocumentFactory.ts +++ b/udmif/event-handler/src/DeviceDocumentFactory.ts @@ -1,11 +1,16 @@ import { DeviceDocumentBuilder, DeviceDocument } from './DeviceDocument'; -import { isPointset, isSystem } from './DocumentTypeUtil'; +import { + isPointset, + isSystem, +} from './DocumentTypeUtil'; import { UdmiMessage } from './UdmiMessage'; import { PointBuilder, Point } from './Point'; export function createDeviceDocument(udmiMessage: UdmiMessage, existingPoints: Point[]): DeviceDocument { const builder: DeviceDocumentBuilder = new DeviceDocumentBuilder(); - builder.id(udmiMessage.attributes.deviceNumId).name(udmiMessage.attributes.deviceId); + builder + .id(udmiMessage.attributes.deviceNumId) + .name(udmiMessage.attributes.deviceId); if (isSystem(udmiMessage)) { return buildDeviceDocumentFromSystem(udmiMessage, builder); @@ -14,7 +19,10 @@ export function createDeviceDocument(udmiMessage: UdmiMessage, existingPoints: P } } -function buildDeviceDocumentFromSystem(udmiMessage: UdmiMessage, builder: DeviceDocumentBuilder): DeviceDocument { +function buildDeviceDocumentFromSystem( + udmiMessage: UdmiMessage, + builder: DeviceDocumentBuilder +): DeviceDocument { return builder .lastPayload(udmiMessage.data.timestamp) .operational(udmiMessage.data.operational) @@ -35,7 +43,7 @@ function buildDeviceDocumentFromPointset( const points: Point[] = []; for (var pointCode in udmiMessage.data.points) { - const existingPoint = existingPoints.find((point) => point.name === pointCode); + const existingPoint = existingPoints.find(point => point.name === pointCode); const point: Point = buildPoint(udmiMessage, existingPoint, pointCode); points.push(point); } @@ -44,6 +52,7 @@ function buildDeviceDocumentFromPointset( } function buildPoint(udmiMessage: UdmiMessage, existingPoint: Point, pointCode: string): Point { + const pointValue = udmiMessage.data.points[pointCode]; // we get the value from either the message or the existing point @@ -61,3 +70,4 @@ function buildPoint(udmiMessage: UdmiMessage, existingPoint: Point, pointCode: s .metaCode(pointCode) .build(); } + diff --git a/udmif/event-handler/src/DocumentTypeUtil.ts b/udmif/event-handler/src/DocumentTypeUtil.ts index 4e04e84c4e..34e4f7438f 100644 --- a/udmif/event-handler/src/DocumentTypeUtil.ts +++ b/udmif/event-handler/src/DocumentTypeUtil.ts @@ -15,3 +15,5 @@ export function isSystem(message): boolean { export function isSubFolder(message, folderName: string): boolean { return message.attributes.subFolder === folderName; } + + diff --git a/udmif/event-handler/src/UdmiMessageHandler.ts b/udmif/event-handler/src/UdmiMessageHandler.ts index d07ee707f7..db06311d95 100644 --- a/udmif/event-handler/src/UdmiMessageHandler.ts +++ b/udmif/event-handler/src/UdmiMessageHandler.ts @@ -5,7 +5,7 @@ import { UdmiMessage } from './UdmiMessage'; import { DeviceKey } from './DeviceKey'; export default class UdmiMessageHandler { - constructor(private deviceDao: DeviceDao) {} + constructor(private deviceDao: DeviceDao) { } async handleUdmiEvent(udmiMessage: UdmiMessage) { try { diff --git a/udmif/event-handler/src/__tests__/DeviceDao.spec.ts b/udmif/event-handler/src/__tests__/DeviceDao.spec.ts index ecf12c4d40..f7640b970b 100644 --- a/udmif/event-handler/src/__tests__/DeviceDao.spec.ts +++ b/udmif/event-handler/src/__tests__/DeviceDao.spec.ts @@ -23,7 +23,7 @@ afterAll(async () => { beforeEach(async () => { // clean the collection before each test await deviceCollection.deleteMany({}); -}); +}) describe('DeviceDao.upsert', () => { test('upsert calls the updateOne method on the provided collection', () => { @@ -44,6 +44,7 @@ describe('DeviceDao.upsert', () => { }); describe('DeviceDao.get', () => { + test('get method is called and returns the matching document', async () => { // arrange const findOneSpy = jest.spyOn(deviceCollection, 'findOne'); @@ -63,4 +64,4 @@ describe('DeviceDao.get', () => { expect(retrievedDeviceDocument).toEqual(insertedDeviceDocument); expect(findOneSpy).toHaveBeenCalledWith(deviceKey); }); -}); +}); \ No newline at end of file diff --git a/udmif/event-handler/src/__tests__/DeviceDaoFactory.spec.ts b/udmif/event-handler/src/__tests__/DeviceDaoFactory.spec.ts index 4f7955b52b..68ce119e10 100644 --- a/udmif/event-handler/src/__tests__/DeviceDaoFactory.spec.ts +++ b/udmif/event-handler/src/__tests__/DeviceDaoFactory.spec.ts @@ -23,6 +23,7 @@ describe('DeviceDaoFactory.getDeviceDAO()', () => { }); describe('DeviceDaoFactory.getUri()', () => { + // take a backup of the environment prior to running the tests const ENV_BACKUP = { ...process.env }; diff --git a/udmif/event-handler/src/__tests__/DeviceDocument.spec.ts b/udmif/event-handler/src/__tests__/DeviceDocument.spec.ts index a6217a9ce2..2d4f6a0fc7 100644 --- a/udmif/event-handler/src/__tests__/DeviceDocument.spec.ts +++ b/udmif/event-handler/src/__tests__/DeviceDocument.spec.ts @@ -60,7 +60,7 @@ describe('DeviceDocument.DeviceDocumentBuilder', () => { site, serialNumber, points, - tags, + tags }); }); }); diff --git a/udmif/event-handler/src/__tests__/DeviceDocumentFactory.spec.ts b/udmif/event-handler/src/__tests__/DeviceDocumentFactory.spec.ts index 0837e55e9f..3cbd8cd3a2 100644 --- a/udmif/event-handler/src/__tests__/DeviceDocumentFactory.spec.ts +++ b/udmif/event-handler/src/__tests__/DeviceDocumentFactory.spec.ts @@ -1,6 +1,12 @@ import { createDeviceDocument } from '../DeviceDocumentFactory'; import { DeviceDocument } from '../DeviceDocument'; -import { CONFIG, MODEL, POINTSET_SUB_FOLDER, STATE, SYSTEM_SUB_FOLDER } from '../DocumentTypeUtil'; +import { + CONFIG, + MODEL, + POINTSET_SUB_FOLDER, + STATE, + SYSTEM_SUB_FOLDER, +} from '../DocumentTypeUtil'; import { UdmiMessage } from '../UdmiMessage'; import { Point } from '../Point'; @@ -10,6 +16,7 @@ const BASIC_SYSTEM_ATTRIBUTES = { deviceId: name, deviceNumId: id, subFolder: SY const BASIC_POINTSET_ATTRIBUTES = { deviceId: name, deviceNumId: id, subFolder: POINTSET_SUB_FOLDER }; describe('DeviceDocumentFactory.createDeviceDocument.default', () => { + const points: Point[] = []; const tags: string[] = []; @@ -77,9 +84,10 @@ describe('DeviceDocumentFactory.createDeviceDocument.pointset', () => { beforeEach(() => { existingPoints = []; - }); + }) test('creates a device document with pointset', () => { + // arrange const inputMessage: UdmiMessage = { attributes: { ...BASIC_POINTSET_ATTRIBUTES }, @@ -106,6 +114,7 @@ describe('DeviceDocumentFactory.createDeviceDocument.pointset', () => { }); test('creates a device document with pointset model', () => { + const inputMessage: UdmiMessage = { attributes: { ...BASIC_POINTSET_ATTRIBUTES, subType: MODEL }, data: { @@ -187,8 +196,11 @@ describe('DeviceDocumentFactory.createDeviceDocument.pointset', () => { }); test('merges a device document with pointset', () => { + // existing point has units and a value - existingPoints.push({ name: faps, id: faps, value: '70', units: 'Bars', meta: { code: faps, units: 'Bars' } }); + existingPoints.push( + { name: faps, id: faps, value: '70', units: 'Bars', meta: { code: faps, units: 'Bars' } } + ) // arrange const inputMessage: UdmiMessage = { @@ -214,4 +226,6 @@ describe('DeviceDocumentFactory.createDeviceDocument.pointset', () => { // act and assert expect(createDeviceDocument(inputMessage, existingPoints)).toEqual(expectedDeviceDocument); }); + }); + diff --git a/udmif/event-handler/src/__tests__/DeviceTypeUtils.spec.ts b/udmif/event-handler/src/__tests__/DeviceTypeUtils.spec.ts index e63d9ce1ab..3a48532cba 100644 --- a/udmif/event-handler/src/__tests__/DeviceTypeUtils.spec.ts +++ b/udmif/event-handler/src/__tests__/DeviceTypeUtils.spec.ts @@ -24,6 +24,7 @@ describe('DeviceTypeUtils.isSystemState', () => { const inputMessage: UdmiMessage = { attributes: { ...BASIC_SYSTEM_ATTRIBUTES, subType: MODEL }, data: {} }; expect(isSystem(inputMessage)).toEqual(true); }); + }); describe('DeviceDocumentFactory.isPointset...', () => { @@ -47,3 +48,4 @@ describe('DeviceDocumentFactory.isPointset...', () => { expect(isPointset(inputMessage)).toEqual(true); }); }); + diff --git a/udmif/event-handler/src/__tests__/Points.spec.ts b/udmif/event-handler/src/__tests__/Points.spec.ts index e6251779c9..3cc843d900 100644 --- a/udmif/event-handler/src/__tests__/Points.spec.ts +++ b/udmif/event-handler/src/__tests__/Points.spec.ts @@ -33,18 +33,13 @@ describe('Points.PointBuilder', () => { expect(builder.build()).toEqual({ id, name, state, units, value }); }); test('Builder allows optional meta attributes', () => { - builder - .id(id) - .name('filter_alarm_pressure_status') - .units('Degrees-Celsius') - .metaCode('filter_alarm_pressure_status') - .metaUnit('Degrees-Celsius'); + builder.id(id).name('filter_alarm_pressure_status').units('Degrees-Celsius').metaCode('filter_alarm_pressure_status').metaUnit('Degrees-Celsius'); expect(builder.build()).toEqual({ id, name: 'filter_alarm_pressure_status', units: 'Degrees-Celsius', meta: { code: 'filter_alarm_pressure_status', units: 'Degrees-Celsius' }, - state: '', + state: '' }); }); }); diff --git a/udmif/web/src/app/app.module.ts b/udmif/web/src/app/app.module.ts index 36880444b3..0f3294cb45 100644 --- a/udmif/web/src/app/app.module.ts +++ b/udmif/web/src/app/app.module.ts @@ -21,4 +21,4 @@ import { EnvModule } from './env/env.module'; ], bootstrap: [AppComponent], }) -export class AppModule {} +export class AppModule { } diff --git a/udmif/web/src/app/auth/auth.module.ts b/udmif/web/src/app/auth/auth.module.ts index d5164aef03..acac42819c 100644 --- a/udmif/web/src/app/auth/auth.module.ts +++ b/udmif/web/src/app/auth/auth.module.ts @@ -26,4 +26,4 @@ export function SocialAuthFactory(env: EnvService): SocialAuthServiceConfig { }, ], }) -export class AuthModule {} +export class AuthModule { } diff --git a/udmif/web/src/app/auth/auth.service.ts b/udmif/web/src/app/auth/auth.service.ts index 1375b71176..7128b059bb 100644 --- a/udmif/web/src/app/auth/auth.service.ts +++ b/udmif/web/src/app/auth/auth.service.ts @@ -24,7 +24,7 @@ export class AuthService { try { await this.socialAuth.signIn(GoogleLoginProvider.PROVIDER_ID); this.router.navigateByUrl('/devices'); - } catch {} + } catch { } } async logout(): Promise { @@ -35,7 +35,7 @@ export class AuthService { // and the AuthService needs to makes use of the Apollo service, // we need to use the injector.get pattern to avoid circular dep issue. this.injector.get(Apollo).client.clearStore(); - } catch {} + } catch { } } async refreshToken(): Promise { diff --git a/udmif/web/src/app/graphql/graphql.module.ts b/udmif/web/src/app/graphql/graphql.module.ts index 9beb6349d1..86e2d9c2f7 100644 --- a/udmif/web/src/app/graphql/graphql.module.ts +++ b/udmif/web/src/app/graphql/graphql.module.ts @@ -68,4 +68,4 @@ export function ApolloFactory(httpLink: HttpLink, env: EnvService, auth: AuthSer }, ], }) -export class GraphQLModule {} +export class GraphQLModule { } From eb320563ad7999b82dc91b446a23f8d597181c0d Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Mon, 8 Aug 2022 17:31:45 -0700 Subject: [PATCH 17/27] Updating with simpler model --- docs/specs/onboarding.md | 100 +++++++++++++++++---------------------- 1 file changed, 44 insertions(+), 56 deletions(-) diff --git a/docs/specs/onboarding.md b/docs/specs/onboarding.md index 07fa6a6ff3..f86d3c9fac 100644 --- a/docs/specs/onboarding.md +++ b/docs/specs/onboarding.md @@ -12,57 +12,52 @@ scopes of device data: * **[Discovery](discovery.md)**: Messages relating to the discovery (and provisioning) of devices (e.g. messy BACnet info) * **[Mapping](mapping.md)**: Messages relating to a 'resolved' device type and ID (e.g. the device is an `AHU` called `AHU-1`) * **[Pointset](../messages/pointset.md)**: Messages relating to actual data flow (e.g. temperature reading), essentially the interesting stuff +* **(Onboard)**: Interactions with an external (non-UDMI) entity of some kind to facilitate onboarding of devices ## Sequence Diagram The overall onboarding sequence involves multiple components that work together to provide the overall flow: -* **Device**: The target thing that needs to be discovered, configured, and ultimately communications point data +* **Devices**: The target things that need to be discovered, configured, and ultimately communicate point data. * **Spotter**: A on-prem node responsible for handling discovery scans of devices. -* **Cloud**: The on-prem/in-cloud boundary. Things to the left are things in the building, to the right are in the cloud -* **Agent**: Responsible for managing the overall _discovery_ process (how often, what color, etc...) -* **Mapper**: Uses hueristics, ML, or a UI to convert discovery information into a concrete device/sink mapping -* **Sink**: Ultimate recepient of pointset information. The thing that cares about 'temperature' in a room +* **Agent**: Agent responsible for managing the overall _discovery_ and _mapping_ process (how often, what color, etc...). +* **Engine**: Mapping engine that uses hueristics, ML, or a UI to convert discovery information into a concrete device/sink mapping. +* **Sink**: Ultimate recepient of pointset information, The thing that cares about 'temperature' in a room. Notes & Caveats: 1. Only "interesting" messages are shown in the diagram, there's other control flow things that go on (e.g. to configure when the *Spotter* should activate) to complete the overall flow. -2. This just shows one-of-many potential provisioning (handling keys) techniques. There's other paths -that would be possible (including manually, which is the baseline default). -3. This shows the flow for a direct-connect (no IoT Gateway) device. The overall flow for a proxied device -(with IoT Gateway) would more or less be the same, just different details about exact communication mechanisms. +2. This shows the flow for a direct-connect (no IoT Gateway involved) device. The overall flow for a proxied device +(with IoT Gateway) would more or less be the same with some additional intermediaries. ``` -+---------+ +---------+ +-------+ +-------+ +--------+ +-------+ -| Device | | Spotter | | Cloud | | Agent | | Mapper | | Sink | -+---------+ +---------+ +-------+ +-------+ +--------+ +-------+ - | | | | | | - | (Info) | | | | | - |----------------------->| | | | | - | | | | | | - | | Discovery Event | | | - | |------------------------------------------------------->| | - | | | | | | - | | | | | Mapping Event | - | | | | |------------------>| - | | | | | | - | | | | Mapping Event | | - | | | |<-------------------| | - | | | | | | - | | | (Cloud Provision) | | | - | | |<----------------------| | | - | | | | | | - | | | Discovery Command | | | - | |<----------------------------------| | | - | | | | | | - | (Device Provision) | | | | | - |<-----------------------| | | | | - | | | | | | - | | | | | Pointset Config | - |<----------------------------------------------------------------------------------------------------| - | | | | | | - | Pointset Event | | | | | - |---------------------------------------------------------------------------------------------------->| - | | | | | | ++---------+ +---------+ +-------+ +---------+ +-------+ +| Devices | | Spotter | | Agent | | Engine | | Sink | ++---------+ +---------+ +-------+ +---------+ +-------+ + | | | | | + | | Discovery Request | | | + | |<----------------------| | | + | | | | | + | Probes | | | | + |<-------------| | | | + | | | | | + | Replies | | | | + |------------->| | | | + | | | | | + | | Discovery Responses | | | + | |----------------------------------------------->| | + | | | | | + | | | Mapping Request | | + | | |----------------------->| | + | | | | | + | | | Mapping Responses | | + | | |<-----------------------| | + | | | | | + | | | Onboard Requests | | + | | |---------------------------------->| + | | | | | + | Telemetry Events | | | + |------------------------------------------------------------------------->| + | | | | | ``` 1. **(Info)** contains natively-encoded _Device_ information (format out of scope for UDMI) @@ -71,28 +66,21 @@ that would be possible (including manually, which is the baseline default). * "Device `78F936` has points { }, with a public key `XYZZYZ`" 3. **Mapping Event** from the _Mapper_ (recieved by both _Sink_ and _Agent_) * "Device `78F936` is an `AHU` called `AHU-183`, and `room_temp` is really a `flow_temperatue`" -5. **(Cloud Provision)** from _Agent_ sets up the _Cloud_ layer using the IoT Core API. - * "Device `AHU-183` exists and has public key `XYZZYZ`" 4. **[Discovery Command](../../tests/command_discovery.tests/provision.json)** from the _Agent_ to the _Spotter_ contains information necessary to provision the device * "Device `78F936` should call itself `AHU-183` when connecting to the cloud" -5. **(Device Provision)** uses some natively-encoded mechanism for setting up the device with relevant cloud info - * "Device `78F936`, you are celled `AHU-183` when connecting to the cloud" -7. **[Pointset Config](../../tests/config.tests/example.json)** from the _Sink_ can go directly to the _Device_ (after it connects to the cloud) - * "Device `AHU-183`, you should send the `room_temp` data point every `10 minutes`" 8. **[Pointset Events](../../tests/event_pointset.tests/example.json)** sends telemetry events from the _Device_ to _Sink_... business as usual! * "I am `AHU-183`, and my `room_temp` is `73`" ## Source Created using https://textart.io/sequence# ``` -object Device Spotter Cloud Agent Mapper Sink -Device->Spotter: (Info) -Spotter->Mapper: Discovery Event -Mapper->Sink: Mapping Event -Mapper->Agent: Mapping Event -Agent->Cloud: (Cloud Provision) -Agent->Spotter: Discovery Command -Spotter->Device: (Device Provision) -Sink->Device: Pointset Config -Device->Sink: Pointset Event +object Devices Spotter Agent Engine Sink +Agent->Spotter: Discovery Request +Spotter->Devices: Probes +Devices->Spotter: Replies +Spotter->Engine: Discovery Responses +Agent->Engine: Mapping Request +Engine->Agent: Mapping Responses +Agent->Sink: Onboard Requests +Devices->Sink: Telemetry Events ``` From 5168ada0c10c8e82518ccb968128c6001bffd547 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Mon, 8 Aug 2022 21:17:20 -0700 Subject: [PATCH 18/27] Update text --- docs/specs/onboarding.md | 83 +++++++++++++------------ tests/config_mapping.tests/mapping.json | 4 +- 2 files changed, 44 insertions(+), 43 deletions(-) diff --git a/docs/specs/onboarding.md b/docs/specs/onboarding.md index f86d3c9fac..d43265368a 100644 --- a/docs/specs/onboarding.md +++ b/docs/specs/onboarding.md @@ -3,10 +3,9 @@ # Onboarding The overall "onboarding" flow consists of a number of separate subflows stitched together for a complete -end-to-end process. This generall starts from an "unknown" device in the system through to a UDMI-compliant -device that's properly integrated with backend services. +end-to-end process to take an "unknown" device and ensure that it's properly integrated with backend services. -At a high-level, the overall process involves different message subgroups that handle slightly different +At a high-level, the process involves different message subgroups that handle slightly different scopes of device data: * **(Native)**: Device communicaiton using some non-UDMI native protocol (e.g. BACnet, Modbus, etc...) * **[Discovery](discovery.md)**: Messages relating to the discovery (and provisioning) of devices (e.g. messy BACnet info) @@ -30,57 +29,59 @@ to configure when the *Spotter* should activate) to complete the overall flow. (with IoT Gateway) would more or less be the same with some additional intermediaries. ``` -+---------+ +---------+ +-------+ +---------+ +-------+ -| Devices | | Spotter | | Agent | | Engine | | Sink | -+---------+ +---------+ +-------+ +---------+ +-------+ - | | | | | - | | Discovery Request | | | - | |<----------------------| | | - | | | | | - | Probes | | | | - |<-------------| | | | - | | | | | - | Replies | | | | - |------------->| | | | - | | | | | - | | Discovery Responses | | | - | |----------------------------------------------->| | - | | | | | - | | | Mapping Request | | - | | |----------------------->| | - | | | | | - | | | Mapping Responses | | - | | |<-----------------------| | - | | | | | - | | | Onboard Requests | | - | | |---------------------------------->| - | | | | | - | Telemetry Events | | | - |------------------------------------------------------------------------->| - | | | | | ++---------+ +---------+ +-------+ +---------+ +-------+ +| Devices | | Spotter | | Agent | | Engine | | Sink | ++---------+ +---------+ +-------+ +---------+ +-------+ + | | | | | + | | Discovery Config | | | + | |<---------------------| | | + | | | | | + | Probes | | | | + |<-------------| | | | + | | | | | + | Replies | | | | + |------------->| | | | + | | | | | + | | Discovery Events | | | + | |------------------------------------------->| | + | | | | | + | | | Mapping Config | | + | | |-------------------->| | + | | | | | + | | | Mapping Events | | + | | |<--------------------| | + | | | | | + | | | Onboard Requests | | + | | |------------------------------->| + | | | | | + | Telemetry Events | | | + |--------------------------------------------------------------------->| + | | | | | ``` -1. **(Info)** contains natively-encoded _Device_ information (format out of scope for UDMI) +1. **[Discovery Config](../../tests/config.tests/discovery.json)** indicates that the _spotter_ should do the needful and scan the local network. +1. **(Probes)** scan for natively-encoded _device_ information from devices (format out of scope for UDMI) +2. **(Deplies)** contain the discovered natively-encoded _device_ information, e.g.: * "I am device `78F936`, with points { `room_temp`, `step_size`, and `operation_count` }, and public key `XYZZYZ`" -2. **[Discovery Event](../../tests/event_discovery.tests/enumeration.json)** from _Spotter_ wraps up the device info into a UDMI-normalized format +2. **[Discovery Events](../../tests/event_discovery.tests/enumeration.json)** wraps the device info from the _spotter_ into a UDMI-normalized format, e.g.: * "Device `78F936` has points { }, with a public key `XYZZYZ`" -3. **Mapping Event** from the _Mapper_ (recieved by both _Sink_ and _Agent_) +3. **[Mapping Config](../../tests/config_mapping.tests/mapping.json)** from the _agent_ indicates that the _engine_ should export responses. +3. **[Mapping Events](../../tests/event_mapping.tests/mapping.json)** from the _engine_ contain actual calculated point mappings. * "Device `78F936` is an `AHU` called `AHU-183`, and `room_temp` is really a `flow_temperatue`" -4. **[Discovery Command](../../tests/command_discovery.tests/provision.json)** from the _Agent_ to the _Spotter_ contains information necessary to provision the device - * "Device `78F936` should call itself `AHU-183` when connecting to the cloud" -8. **[Pointset Events](../../tests/event_pointset.tests/example.json)** sends telemetry events from the _Device_ to _Sink_... business as usual! +4. **Onboard Requests** are requests to the appropriate _sink_ to onboard a device (contents are defined by _sink_ and out of scope for UDMI) +8. **[Telemetry Events](../../tests/event_pointset.tests/example.json)** are data events from _device_ to _sink_... business as usual! * "I am `AHU-183`, and my `room_temp` is `73`" ## Source Created using https://textart.io/sequence# ``` object Devices Spotter Agent Engine Sink -Agent->Spotter: Discovery Request +Agent->Spotter: Discovery Config Spotter->Devices: Probes Devices->Spotter: Replies -Spotter->Engine: Discovery Responses -Agent->Engine: Mapping Request -Engine->Agent: Mapping Responses +Spotter->Engine: Discovery Events +Agent->Engine: Mapping Config +Engine->Agent: Mapping Events Agent->Sink: Onboard Requests Devices->Sink: Telemetry Events ``` diff --git a/tests/config_mapping.tests/mapping.json b/tests/config_mapping.tests/mapping.json index d3b25ae1b1..e2b9f7f01f 100644 --- a/tests/config_mapping.tests/mapping.json +++ b/tests/config_mapping.tests/mapping.json @@ -1,7 +1,7 @@ // // This message represents the entire high-level config of a recommendation model, -// as given to the mapping component. It therefore reflects the external (to mapper) -// view of what has happened outside of the mapper itself. +// as given to the mapping component. It therefore reflects the external (to a mapping engine) +// view of what has happened outside of the mapping engine itself. // // The building id is defined by the message envelope. // From f7e8994fba41acb1b19530573de2cc85a6b8ee2f Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Tue, 9 Aug 2022 07:25:15 -0700 Subject: [PATCH 19/27] Refactor SiteModel errors --- .../com/google/daq/mqtt/util/SiteModel.java | 113 +++++++++++++++ pubber/src/main/java/daq/pubber/Pubber.java | 135 ++++++------------ 2 files changed, 157 insertions(+), 91 deletions(-) create mode 100644 pubber/src/main/java/com/google/daq/mqtt/util/SiteModel.java diff --git a/pubber/src/main/java/com/google/daq/mqtt/util/SiteModel.java b/pubber/src/main/java/com/google/daq/mqtt/util/SiteModel.java new file mode 100644 index 0000000000..6f99351e95 --- /dev/null +++ b/pubber/src/main/java/com/google/daq/mqtt/util/SiteModel.java @@ -0,0 +1,113 @@ +package com.google.daq.mqtt.util; + +import static java.util.stream.Collectors.toMap; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.util.ISO8601DateFormat; +import com.google.common.base.Preconditions; +import daq.pubber.EndpointConfiguration; +import java.io.File; +import java.util.Arrays; +import java.util.Map; +import java.util.Set; +import java.util.function.BiConsumer; +import java.util.stream.Collectors; +import udmi.schema.CloudModel.Auth_type; +import udmi.schema.GatewayModel; +import udmi.schema.Metadata; + +public class SiteModel { + + private static final String KEY_SITE_PATH_FORMAT = "%s/devices/%s/%s_private.pkcs8"; + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper() + .enable(SerializationFeature.INDENT_OUTPUT) + .setDateFormat(new ISO8601DateFormat()) + .setSerializationInclusion(JsonInclude.Include.NON_NULL); + + final String sitePath; + private Map allMetadata; + private EndpointConfiguration endpointConfig; + + public SiteModel(String sitePath) { + this.sitePath = sitePath; + } + + public static EndpointConfiguration extractEndpointConfig(CloudIotConfig cloudIotConfig) { + EndpointConfiguration endpoint = new EndpointConfiguration(); + endpoint.registryId = cloudIotConfig.registry_id; + endpoint.cloudRegion = cloudIotConfig.cloud_region; + return endpoint; + } + + private Set getAllDevices() { + Preconditions.checkState(sitePath != null, "sitePath not defined"); + File devicesFile = new File(new File(sitePath), "devices"); + File[] files = Preconditions.checkNotNull(devicesFile.listFiles(), "no files in site devices/"); + return Arrays.stream(files).map(File::getName).collect(Collectors.toSet()); + } + + private void loadAllDeviceMetadata() { + allMetadata = getAllDevices().stream().collect(toMap(key -> key, this::loadDeviceMetadata)); + } + + private Metadata loadDeviceMetadata(String deviceId) { + Preconditions.checkState(sitePath != null, "sitePath not defined"); + File devicesFile = new File(new File(sitePath), "devices"); + File deviceDir = new File(devicesFile, deviceId); + File deviceMetadataFile = new File(deviceDir, "metadata.json"); + try { + return OBJECT_MAPPER.readValue(deviceMetadataFile, Metadata.class); + } catch (Exception e) { + throw new RuntimeException( + "While reading metadata file " + deviceMetadataFile.getAbsolutePath(), e); + } + } + + public Metadata getMetadata(String deviceId) { + return allMetadata.get(deviceId); + } + + public void forEachDevice(BiConsumer consumer) { + allMetadata.forEach(consumer); + } + + private void loadEndpointConfig() { + Preconditions.checkState(sitePath != null, + "sitePath not defined in configuration"); + File cloudConfig = new File(new File(sitePath), "cloud_iot_config.json"); + try { + endpointConfig = extractEndpointConfig( + OBJECT_MAPPER.readValue(cloudConfig, CloudIotConfig.class)); + } catch (Exception e) { + throw new RuntimeException("While reading config file " + cloudConfig.getAbsolutePath(), e); + } + } + + public void initialize() { + loadEndpointConfig(); + loadAllDeviceMetadata(); + } + + public Auth_type getAuthType(String deviceId) { + return allMetadata.get(deviceId).cloud.auth_type; + } + + public String getDeviceKeyFile(String deviceId) { + String gatewayId = findGateway(deviceId); + String keyDevice = gatewayId != null ? gatewayId : deviceId; + return String.format(KEY_SITE_PATH_FORMAT, sitePath, + keyDevice, getDeviceKeyPrefix(keyDevice)); + } + + private String findGateway(String deviceId) { + GatewayModel gateway = getMetadata(deviceId).gateway; + return gateway != null ? gateway.gateway_id : null; + } + + private String getDeviceKeyPrefix(String targetId) { + Auth_type auth_type = getMetadata(targetId).cloud.auth_type; + return auth_type.value().startsWith("RS") ? "rsa" : "ec"; + } +} diff --git a/pubber/src/main/java/daq/pubber/Pubber.java b/pubber/src/main/java/daq/pubber/Pubber.java index ec121172ca..d68fcf6ce9 100644 --- a/pubber/src/main/java/daq/pubber/Pubber.java +++ b/pubber/src/main/java/daq/pubber/Pubber.java @@ -12,6 +12,7 @@ import com.google.common.collect.ImmutableSet; import com.google.daq.mqtt.util.CatchingScheduledThreadPoolExecutor; import com.google.daq.mqtt.util.CloudIotConfig; +import com.google.daq.mqtt.util.SiteModel; import daq.pubber.MqttPublisher.PublisherException; import daq.pubber.PubSubClient.Bundle; import daq.pubber.SwarmMessage.Attributes; @@ -24,7 +25,6 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.time.Instant; -import java.util.Arrays; import java.util.Base64; import java.util.Date; import java.util.HashMap; @@ -93,7 +93,6 @@ public class Pubber { private static final int DEFAULT_REPORT_SEC = 10; private static final int CONFIG_WAIT_TIME_MS = 10000; private static final int STATE_THROTTLE_MS = 2000; - private static final String KEY_SITE_PATH_FORMAT = "%s/devices/%s/%s_private.pkcs8"; private static final String OUT_DIR = "out"; private static final String PUBSUB_SITE = "PubSub"; private static final String SWARM_SUBFOLDER = "swarm"; @@ -138,9 +137,9 @@ public class Pubber { private PubSubClient pubSubClient; private Consumer onDone; private boolean publishingLog; - private Map allMetadata; private String appliedEndpoint; private EndpointConfiguration extractedEndpoint; + private SiteModel siteModel; /** * Start an instance from a configuration file. @@ -260,63 +259,6 @@ private static void startFeedListener(String projectId, String siteName, String } } - private Set getAllDevices() { - Preconditions.checkState(configuration.sitePath != null, "sitePath not defined"); - File devicesFile = new File(new File(configuration.sitePath), "devices"); - File[] files = Preconditions.checkNotNull(devicesFile.listFiles(), "no files in site devices/"); - return Arrays.stream(files).map(File::getName).collect(Collectors.toSet()); - } - - private void loadDeviceMetadata() { - Preconditions.checkState(configuration.deviceId != null, "deviceId not defined"); - allMetadata = getAllDevices().stream() - .collect(toMap(deviceId -> deviceId, deviceId -> getDeviceMetadata(deviceId))); - processDeviceMetadata(allMetadata.get(configuration.deviceId)); - } - - private Metadata getDeviceMetadata(String deviceId) { - Preconditions.checkState(configuration.sitePath != null, "sitePath not defined"); - File devicesFile = new File(new File(configuration.sitePath), "devices"); - File deviceDir = new File(devicesFile, deviceId); - File deviceMetadataFile = new File(deviceDir, "metadata.json"); - try { - return OBJECT_MAPPER.readValue(deviceMetadataFile, Metadata.class); - } catch (Exception e) { - throw new RuntimeException( - "While reading metadata file " + deviceMetadataFile.getAbsolutePath(), e); - } - } - - private void processDeviceMetadata(Metadata metadata) { - if (metadata.cloud != null) { - configuration.algorithm = metadata.cloud.auth_type.value(); - info("Configuring with key type " + configuration.algorithm); - } - - if (metadata.gateway != null) { - configuration.gatewayId = metadata.gateway.gateway_id; - if (configuration.gatewayId != null) { - Auth_type authType = allMetadata.get(configuration.gatewayId).cloud.auth_type; - if (authType != null) { - configuration.algorithm = authType.value(); - } - } - } - - Map points = - metadata.pointset == null ? DEFAULT_POINTS : metadata.pointset.points; - - if (configuration.options.missingPoint != null) { - if (points.containsKey(configuration.options.missingPoint)) { - points.remove(configuration.options.missingPoint); - } else { - throw new RuntimeException("missingPoint not in pointset"); - } - } - - points.forEach((name, point) -> addPoint(makePoint(name, point))); - } - private AbstractPoint makePoint(String name, PointPointsetModel point) { boolean writable = point.writable != null && point.writable; if (BOOLEAN_UNITS.contains(point.units)) { @@ -343,22 +285,6 @@ private double convertValue(Object baselineValue, double defaultBaselineValue) { throw new RuntimeException("Unknown value type " + baselineValue.getClass()); } - private void loadCloudConfig() { - Preconditions.checkState(configuration.sitePath != null, - "sitePath not defined in configuration"); - File cloudConfig = new File(new File(configuration.sitePath), "cloud_iot_config.json"); - try { - processCloudConfig(OBJECT_MAPPER.readValue(cloudConfig, CloudIotConfig.class)); - } catch (Exception e) { - throw new RuntimeException("While reading config file " + cloudConfig.getAbsolutePath(), e); - } - } - - private void processCloudConfig(CloudIotConfig cloudIotConfig) { - configuration.endpoint.registryId = cloudIotConfig.registry_id; - configuration.endpoint.cloudRegion = cloudIotConfig.cloud_region; - } - private void initializeDevice() { deviceState.system = new SystemState(); deviceState.pointset = new PointsetState(); @@ -367,8 +293,9 @@ private void initializeDevice() { devicePoints.points = new HashMap<>(); if (configuration.sitePath != null) { - loadCloudConfig(); - loadDeviceMetadata(); + siteModel = new SiteModel(configuration.sitePath); + siteModel.initialize(); + processDeviceMetadata(siteModel.getMetadata(configuration.deviceId)); } else if (pubSubClient != null) { pullDeviceMessage(); } @@ -450,8 +377,41 @@ private void processSwarmConfig(SwarmMessage swarm, SwarmMessage.Attributes attr configuration.deviceId = Preconditions.checkNotNull(attributes.deviceId, "deviceId"); configuration.keyBytes = Base64.getDecoder() .decode(Preconditions.checkNotNull(swarm.key_base64, "key_base64")); - processCloudConfig(makeCloudIotConfig(attributes)); - processDeviceMetadata(Preconditions.checkNotNull(swarm.device_metadata, "device_metadata")); + CloudIotConfig cloudIotConfig = makeCloudIotConfig(attributes); + configuration.endpoint = SiteModel.extractEndpointConfig(cloudIotConfig); + processDeviceMetadata( + Preconditions.checkNotNull(swarm.device_metadata, "device_metadata")); + } + + + public void processDeviceMetadata(Metadata metadata) { + if (metadata.cloud != null) { + configuration.algorithm = metadata.cloud.auth_type.value(); + info("Configuring with key type " + configuration.algorithm); + } + + if (metadata.gateway != null) { + configuration.gatewayId = metadata.gateway.gateway_id; + if (configuration.gatewayId != null) { + Auth_type authType = siteModel.getAuthType(configuration.gatewayId); + if (authType != null) { + configuration.algorithm = authType.value(); + } + } + } + + Map points = + metadata.pointset == null ? DEFAULT_POINTS : metadata.pointset.points; + + if (configuration.options.missingPoint != null) { + if (points.containsKey(configuration.options.missingPoint)) { + points.remove(configuration.options.missingPoint); + } else { + throw new RuntimeException("missingPoint not in pointset"); + } + } + + points.forEach((name, point) -> addPoint(makePoint(name, point))); } private CloudIotConfig makeCloudIotConfig(Attributes attributes) { @@ -603,11 +563,8 @@ private void disconnectMqtt() { private void initializeMqtt() { Preconditions.checkNotNull(configuration.deviceId, "configuration deviceId not defined"); - if (configuration.sitePath != null && configuration.keyFile != null) { - String keyDevice = - configuration.gatewayId != null ? configuration.gatewayId : configuration.deviceId; - configuration.keyFile = String.format(KEY_SITE_PATH_FORMAT, configuration.sitePath, - keyDevice, getDeviceKeyPrefix()); + if (siteModel != null && configuration.keyFile != null) { + configuration.keyFile = siteModel.getDeviceKeyFile(configuration.deviceId); } Preconditions.checkState(mqttPublisher == null, "mqttPublisher already defined"); ensureKeyBytes(); @@ -641,10 +598,6 @@ private void ensureKeyBytes() { configuration.keyFile = null; } - private String getDeviceKeyPrefix() { - return configuration.algorithm.startsWith("RS") ? "rsa" : "ec"; - } - private void connect() { try { mqttPublisher.connect(configuration.deviceId); @@ -840,7 +793,7 @@ private void updateDiscoveryEnumeration(FamilyDiscoveryConfig enumeration) { } private Map enumeratePoints(String deviceId) { - return allMetadata.get(deviceId).pointset.points.entrySet().stream().collect( + return siteModel.getMetadata(deviceId).pointset.points.entrySet().stream().collect( Collectors.toMap(this::getPointUniqKey, this::getPointEnumerationEvent)); } @@ -950,7 +903,7 @@ private void sendDiscoveryEvent(String family, Date scanGeneration) { if (scanGeneration.equals(familyDiscoveryState.generation) && familyDiscoveryState.active) { AtomicInteger sentEvents = new AtomicInteger(); - allMetadata.forEach((deviceId, targetMetadata) -> { + siteModel.forEachDevice((deviceId, targetMetadata) -> { FamilyLocalnetModel familyLocalnetModel = getFamilyLocalnetModel(family, targetMetadata); if (familyLocalnetModel != null && familyLocalnetModel.id != null) { DiscoveryEvent discoveryEvent = new DiscoveryEvent(); From 3b7774b0aa87931dc3e4a814acb6863f047fe778 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Tue, 9 Aug 2022 07:33:39 -0700 Subject: [PATCH 20/27] Extract common utils --- .../src/main/java/com/google/udmi}/util/CloudIotConfig.java | 2 +- .../src/main/java/com/google/udmi}/util/SiteModel.java | 2 +- pubber/build.gradle | 1 + pubber/pubber.iml | 3 +++ pubber/src/main/java/daq/pubber/Pubber.java | 4 ++-- 5 files changed, 8 insertions(+), 4 deletions(-) rename {pubber/src/main/java/com/google/daq/mqtt => common/src/main/java/com/google/udmi}/util/CloudIotConfig.java (88%) rename {pubber/src/main/java/com/google/daq/mqtt => common/src/main/java/com/google/udmi}/util/SiteModel.java (99%) diff --git a/pubber/src/main/java/com/google/daq/mqtt/util/CloudIotConfig.java b/common/src/main/java/com/google/udmi/util/CloudIotConfig.java similarity index 88% rename from pubber/src/main/java/com/google/daq/mqtt/util/CloudIotConfig.java rename to common/src/main/java/com/google/udmi/util/CloudIotConfig.java index 34dd842c95..36aa59ae30 100644 --- a/pubber/src/main/java/com/google/daq/mqtt/util/CloudIotConfig.java +++ b/common/src/main/java/com/google/udmi/util/CloudIotConfig.java @@ -1,4 +1,4 @@ -package com.google.daq.mqtt.util; +package com.google.udmi.util; /** * Configuration parameters for a cloud connection. diff --git a/pubber/src/main/java/com/google/daq/mqtt/util/SiteModel.java b/common/src/main/java/com/google/udmi/util/SiteModel.java similarity index 99% rename from pubber/src/main/java/com/google/daq/mqtt/util/SiteModel.java rename to common/src/main/java/com/google/udmi/util/SiteModel.java index 6f99351e95..61067e64d3 100644 --- a/pubber/src/main/java/com/google/daq/mqtt/util/SiteModel.java +++ b/common/src/main/java/com/google/udmi/util/SiteModel.java @@ -1,4 +1,4 @@ -package com.google.daq.mqtt.util; +package com.google.udmi.util; import static java.util.stream.Collectors.toMap; diff --git a/pubber/build.gradle b/pubber/build.gradle index dd4ce79490..f36a3711ea 100644 --- a/pubber/build.gradle +++ b/pubber/build.gradle @@ -25,6 +25,7 @@ sourceSets { main { java { srcDirs '../gencode/java' + srcDirs '../common/src/main/java' } } } diff --git a/pubber/pubber.iml b/pubber/pubber.iml index fc8439aeae..af75080842 100644 --- a/pubber/pubber.iml +++ b/pubber/pubber.iml @@ -4,6 +4,9 @@ + + + diff --git a/pubber/src/main/java/daq/pubber/Pubber.java b/pubber/src/main/java/daq/pubber/Pubber.java index d68fcf6ce9..ffaad7ef90 100644 --- a/pubber/src/main/java/daq/pubber/Pubber.java +++ b/pubber/src/main/java/daq/pubber/Pubber.java @@ -11,8 +11,8 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.daq.mqtt.util.CatchingScheduledThreadPoolExecutor; -import com.google.daq.mqtt.util.CloudIotConfig; -import com.google.daq.mqtt.util.SiteModel; +import com.google.udmi.util.CloudIotConfig; +import com.google.udmi.util.SiteModel; import daq.pubber.MqttPublisher.PublisherException; import daq.pubber.PubSubClient.Bundle; import daq.pubber.SwarmMessage.Attributes; From 256f38c2866608e94fd4a73cb227a9f36e787b0e Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Tue, 9 Aug 2022 07:37:18 -0700 Subject: [PATCH 21/27] Fix javadoc --- pubber/src/main/java/daq/pubber/Pubber.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pubber/src/main/java/daq/pubber/Pubber.java b/pubber/src/main/java/daq/pubber/Pubber.java index ffaad7ef90..6490bb896e 100644 --- a/pubber/src/main/java/daq/pubber/Pubber.java +++ b/pubber/src/main/java/daq/pubber/Pubber.java @@ -383,8 +383,7 @@ private void processSwarmConfig(SwarmMessage swarm, SwarmMessage.Attributes attr Preconditions.checkNotNull(swarm.device_metadata, "device_metadata")); } - - public void processDeviceMetadata(Metadata metadata) { + private void processDeviceMetadata(Metadata metadata) { if (metadata.cloud != null) { configuration.algorithm = metadata.cloud.auth_type.value(); info("Configuring with key type " + configuration.algorithm); From 874e87fbd2ca6790dfeea434b1a32ade3fe7cdbf Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Tue, 9 Aug 2022 08:01:05 -0700 Subject: [PATCH 22/27] Cleanup projectId handling --- .gencode_hash.txt | 6 ++-- .../java/com/google/udmi/util/SiteModel.java | 26 ++++++++++++-- gencode/docs/envelope.html | 35 ++++++++++++++++++- gencode/java/udmi/schema/Envelope.java | 7 +++- gencode/python/udmi/schema/envelope.py | 4 +++ .../main/java/daq/pubber/Configuration.java | 1 + pubber/src/main/java/daq/pubber/Pubber.java | 30 ++++++---------- .../main/java/daq/pubber/SwarmMessage.java | 8 ----- schema/envelope.json | 5 +++ tests/envelope.tests/example2.json | 1 + 10 files changed, 87 insertions(+), 36 deletions(-) diff --git a/.gencode_hash.txt b/.gencode_hash.txt index 7aad824262..436967489a 100644 --- a/.gencode_hash.txt +++ b/.gencode_hash.txt @@ -1,5 +1,5 @@ f5cf191b0151d2142fa634d1deafb93e35041d5b9ae89b26beab32fa25fb9521 gencode/docs/config.html -e2944b13db5ff06be9caea51d03bca48f2cb093a8bb583dca14051255d34ea6b gencode/docs/envelope.html +90679d3d866579501e7aa00b515af05d42fc9fe399eafacaacf297d1e4a22884 gencode/docs/envelope.html 368f12b9b695b31d97e65509c461f56c12762ec50b9baff89fd7fd85713e8fce gencode/docs/event_discovery.html 8133e380e40f27c56accbffc665b2eeb56ec84a4da3b52ba7aa5e439c9c40572 gencode/docs/event_pointset.html 076eb51e75281ed065a9f0236a1e19b6bf3c277b51ceca84f6aa3f76d5bb6022 gencode/docs/event_system.html @@ -25,7 +25,7 @@ d8a80ab3180d33bfa17564c969018e1d4350a47dbc70c4ae8a5abbfb25cfedc9 gencode/java/u 04112dd47b0f761131c276c67d3cd8b789d25e6716b5732be9fef14fc6831f1d gencode/java/udmi/schema/DiscoveryModel.java 9962b0eb7d5adf52af6160e9f3131f8eeb52ae9e518954dbb6aead1bcad0245e gencode/java/udmi/schema/DiscoveryState.java 090bbaf1508aa6ca8380af936af990673f300eb5a940c9e8ab94deb64efa2b7b gencode/java/udmi/schema/Entry.java -8f71ecd4c32044eccd74225006887d58827976f699ea03efaa5b7ed1b69849d0 gencode/java/udmi/schema/Envelope.java +cd362f94454eba8fd5ce3fce5d5e2b5f046d0dd9c35b01de69ef4d2e38413cc5 gencode/java/udmi/schema/Envelope.java e9f5c77be81486b6b8c6d88f70f2d50583d8c3fafa2ac09ead80f44b8d5e751e gencode/java/udmi/schema/Event.java aa0885ca43ab38c7597eacc38b7c512940a1a9fa061abd47d02c28e66b6fd93e gencode/java/udmi/schema/FamilyDiscoveryConfig.java ae4a645f199c8e24b3303463d428ca17af7603ae9ae9238397a6a82e752ab454 gencode/java/udmi/schema/FamilyDiscoveryEvent.java @@ -83,7 +83,7 @@ b461bdc24310ef972faf579b5be577b5af67fb0977d6afb4c42955211b26e3d5 gencode/python 9eab64849e04b25203d5da47856c3f8dda2b96903e4dc43ab932ee35014700bd gencode/python/udmi/schema/config_pointset.py 607c5047df878a1333df3ce88dcce34668959b0b315f6954bf1a4963dcf7839e gencode/python/udmi/schema/config_pointset_point.py 68685118163ea55f23938aafd10308faad1e2bf86c4f2fa9341cb53e19e2a260 gencode/python/udmi/schema/config_system.py -631371489cb1275517bebcc4040cbc655d18ca147ab540701b3fd9cedba138c5 gencode/python/udmi/schema/envelope.py +998ce105f88686f27b85f3630a396ed04b106f830c133a684ea5c505ca95b1c3 gencode/python/udmi/schema/envelope.py 1eb9019b9d0b4ff7de2df8beb625a4f89292d636ece0c02f160495c537bd6c2c gencode/python/udmi/schema/event.py 82182e3f569ad80dc0751027959c7db9135d10072fbe79f896d63a4cd2f4771f gencode/python/udmi/schema/event_discovery.py ad33b91a7fabb4eed7e49c30a983a2106c96330facbe0f376f94d06e2263d6d0 gencode/python/udmi/schema/event_discovery_family.py diff --git a/common/src/main/java/com/google/udmi/util/SiteModel.java b/common/src/main/java/com/google/udmi/util/SiteModel.java index 61067e64d3..1033e2a187 100644 --- a/common/src/main/java/com/google/udmi/util/SiteModel.java +++ b/common/src/main/java/com/google/udmi/util/SiteModel.java @@ -15,6 +15,7 @@ import java.util.function.BiConsumer; import java.util.stream.Collectors; import udmi.schema.CloudModel.Auth_type; +import udmi.schema.Envelope; import udmi.schema.GatewayModel; import udmi.schema.Metadata; @@ -41,6 +42,20 @@ public static EndpointConfiguration extractEndpointConfig(CloudIotConfig cloudIo return endpoint; } + private static CloudIotConfig makeCloudIotConfig(Envelope attributes) { + CloudIotConfig cloudIotConfig = new CloudIotConfig(); + cloudIotConfig.registry_id = Preconditions.checkNotNull(attributes.deviceRegistryId, + "deviceRegistryId"); + cloudIotConfig.cloud_region = Preconditions.checkNotNull(attributes.deviceRegistryLocation, + "deviceRegistryLocation"); + return cloudIotConfig; + } + + public static EndpointConfiguration makeEndpointConfig(Envelope attributes) { + CloudIotConfig cloudIotConfig = makeCloudIotConfig(attributes); + return extractEndpointConfig(cloudIotConfig); + } + private Set getAllDevices() { Preconditions.checkState(sitePath != null, "sitePath not defined"); File devicesFile = new File(new File(sitePath), "devices"); @@ -73,20 +88,25 @@ public void forEachDevice(BiConsumer consumer) { allMetadata.forEach(consumer); } - private void loadEndpointConfig() { + private void loadEndpointConfig(String projectId) { Preconditions.checkState(sitePath != null, "sitePath not defined in configuration"); File cloudConfig = new File(new File(sitePath), "cloud_iot_config.json"); try { endpointConfig = extractEndpointConfig( OBJECT_MAPPER.readValue(cloudConfig, CloudIotConfig.class)); + endpointConfig.projectId = projectId; } catch (Exception e) { throw new RuntimeException("While reading config file " + cloudConfig.getAbsolutePath(), e); } } - public void initialize() { - loadEndpointConfig(); + public EndpointConfiguration getEndpointConfig() { + return endpointConfig; + } + + public void initialize(String projectId) { + loadEndpointConfig(projectId); loadAllDeviceMetadata(); } diff --git a/gencode/docs/envelope.html b/gencode/docs/envelope.html index 3dce383d13..5a716deadd 100644 --- a/gencode/docs/envelope.html +++ b/gencode/docs/envelope.html @@ -124,6 +124,39 @@

+ + + + +
+
+
+

+ +

+
+ +
+
+ +
root + + + + deviceRegistryLocation
Type: string
+Must match regular expression: ^[a-z]+-[a-z]+1$ + + + + + +
@@ -185,7 +218,7 @@

subFolder

Type: enum (of string)

Must be one of:

-
  • "update"
  • "audit"
  • "discovery"
  • "system"
  • "gateway"
  • "localnet"
  • "pointset"
  • "blobset"
+
  • "update"
  • "audit"
  • "discovery"
  • "system"
  • "gateway"
  • "swarm"
  • "localnet"
  • "pointset"
  • "blobset"
diff --git a/gencode/java/udmi/schema/Envelope.java b/gencode/java/udmi/schema/Envelope.java index 2743a1f0a1..99ac30ca03 100644 --- a/gencode/java/udmi/schema/Envelope.java +++ b/gencode/java/udmi/schema/Envelope.java @@ -22,6 +22,7 @@ "deviceId", "deviceNumId", "deviceRegistryId", + "deviceRegistryLocation", "projectId", "subFolder", "subType" @@ -50,6 +51,8 @@ public class Envelope { */ @JsonProperty("deviceRegistryId") public String deviceRegistryId; + @JsonProperty("deviceRegistryLocation") + public String deviceRegistryLocation; /** * * (Required) @@ -70,6 +73,7 @@ public class Envelope { @Override public int hashCode() { int result = 1; + result = ((result* 31)+((this.deviceRegistryLocation == null)? 0 :this.deviceRegistryLocation.hashCode())); result = ((result* 31)+((this.deviceNumId == null)? 0 :this.deviceNumId.hashCode())); result = ((result* 31)+((this.subFolder == null)? 0 :this.subFolder.hashCode())); result = ((result* 31)+((this.deviceRegistryId == null)? 0 :this.deviceRegistryId.hashCode())); @@ -88,7 +92,7 @@ public boolean equals(Object other) { return false; } Envelope rhs = ((Envelope) other); - return (((((((this.deviceNumId == rhs.deviceNumId)||((this.deviceNumId!= null)&&this.deviceNumId.equals(rhs.deviceNumId)))&&((this.subFolder == rhs.subFolder)||((this.subFolder!= null)&&this.subFolder.equals(rhs.subFolder))))&&((this.deviceRegistryId == rhs.deviceRegistryId)||((this.deviceRegistryId!= null)&&this.deviceRegistryId.equals(rhs.deviceRegistryId))))&&((this.subType == rhs.subType)||((this.subType!= null)&&this.subType.equals(rhs.subType))))&&((this.deviceId == rhs.deviceId)||((this.deviceId!= null)&&this.deviceId.equals(rhs.deviceId))))&&((this.projectId == rhs.projectId)||((this.projectId!= null)&&this.projectId.equals(rhs.projectId)))); + return ((((((((this.deviceRegistryLocation == rhs.deviceRegistryLocation)||((this.deviceRegistryLocation!= null)&&this.deviceRegistryLocation.equals(rhs.deviceRegistryLocation)))&&((this.deviceNumId == rhs.deviceNumId)||((this.deviceNumId!= null)&&this.deviceNumId.equals(rhs.deviceNumId))))&&((this.subFolder == rhs.subFolder)||((this.subFolder!= null)&&this.subFolder.equals(rhs.subFolder))))&&((this.deviceRegistryId == rhs.deviceRegistryId)||((this.deviceRegistryId!= null)&&this.deviceRegistryId.equals(rhs.deviceRegistryId))))&&((this.subType == rhs.subType)||((this.subType!= null)&&this.subType.equals(rhs.subType))))&&((this.deviceId == rhs.deviceId)||((this.deviceId!= null)&&this.deviceId.equals(rhs.deviceId))))&&((this.projectId == rhs.projectId)||((this.projectId!= null)&&this.projectId.equals(rhs.projectId)))); } @Generated("jsonschema2pojo") @@ -99,6 +103,7 @@ public enum SubFolder { DISCOVERY("discovery"), SYSTEM("system"), GATEWAY("gateway"), + SWARM("swarm"), LOCALNET("localnet"), POINTSET("pointset"), BLOBSET("blobset"); diff --git a/gencode/python/udmi/schema/envelope.py b/gencode/python/udmi/schema/envelope.py index a8403cbbca..09a616bea9 100644 --- a/gencode/python/udmi/schema/envelope.py +++ b/gencode/python/udmi/schema/envelope.py @@ -8,6 +8,7 @@ def __init__(self): self.deviceId = None self.deviceNumId = None self.deviceRegistryId = None + self.deviceRegistryLocation = None self.projectId = None self.subFolder = None self.subType = None @@ -20,6 +21,7 @@ def from_dict(source): result.deviceId = source.get('deviceId') result.deviceNumId = source.get('deviceNumId') result.deviceRegistryId = source.get('deviceRegistryId') + result.deviceRegistryLocation = source.get('deviceRegistryLocation') result.projectId = source.get('projectId') result.subFolder = source.get('subFolder') result.subType = source.get('subType') @@ -49,6 +51,8 @@ def to_dict(self): result['deviceNumId'] = self.deviceNumId # 5 if self.deviceRegistryId: result['deviceRegistryId'] = self.deviceRegistryId # 5 + if self.deviceRegistryLocation: + result['deviceRegistryLocation'] = self.deviceRegistryLocation # 5 if self.projectId: result['projectId'] = self.projectId # 5 if self.subFolder: diff --git a/pubber/src/main/java/daq/pubber/Configuration.java b/pubber/src/main/java/daq/pubber/Configuration.java index feae4bd908..7cf8d206db 100644 --- a/pubber/src/main/java/daq/pubber/Configuration.java +++ b/pubber/src/main/java/daq/pubber/Configuration.java @@ -6,6 +6,7 @@ */ public class Configuration { public EndpointConfiguration endpoint = new EndpointConfiguration(); + public String projectId; public String gatewayId; public String deviceId; public String sitePath; diff --git a/pubber/src/main/java/daq/pubber/Pubber.java b/pubber/src/main/java/daq/pubber/Pubber.java index 6490bb896e..fa61cb6993 100644 --- a/pubber/src/main/java/daq/pubber/Pubber.java +++ b/pubber/src/main/java/daq/pubber/Pubber.java @@ -11,11 +11,9 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.daq.mqtt.util.CatchingScheduledThreadPoolExecutor; -import com.google.udmi.util.CloudIotConfig; import com.google.udmi.util.SiteModel; import daq.pubber.MqttPublisher.PublisherException; import daq.pubber.PubSubClient.Bundle; -import daq.pubber.SwarmMessage.Attributes; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.OutputStream; @@ -52,6 +50,8 @@ import udmi.schema.DiscoveryEvent; import udmi.schema.DiscoveryState; import udmi.schema.Entry; +import udmi.schema.Envelope; +import udmi.schema.Envelope.SubFolder; import udmi.schema.FamilyDiscoveryConfig; import udmi.schema.FamilyDiscoveryEvent; import udmi.schema.FamilyDiscoveryState; @@ -95,7 +95,6 @@ public class Pubber { private static final int STATE_THROTTLE_MS = 2000; private static final String OUT_DIR = "out"; private static final String PUBSUB_SITE = "PubSub"; - private static final String SWARM_SUBFOLDER = "swarm"; private static final Set BOOLEAN_UNITS = ImmutableSet.of("No-units"); private static final double DEFAULT_BASELINE_VALUE = 50; private static final String MESSAGE_CATEGORY_FORMAT = "system.%s.%s"; @@ -167,7 +166,7 @@ public Pubber(String configPath) { */ public Pubber(String projectId, String sitePath, String deviceId, String serialNo) { configuration = new Configuration(); - configuration.endpoint.projectId = projectId; + configuration.projectId = projectId; configuration.deviceId = deviceId; configuration.serialNo = serialNo; if (PUBSUB_SITE.equals(sitePath)) { @@ -294,7 +293,8 @@ private void initializeDevice() { if (configuration.sitePath != null) { siteModel = new SiteModel(configuration.sitePath); - siteModel.initialize(); + siteModel.initialize(configuration.projectId); + configuration.endpoint = siteModel.getEndpointConfig(); processDeviceMetadata(siteModel.getMetadata(configuration.deviceId)); } else if (pubSubClient != null) { pullDeviceMessage(); @@ -345,10 +345,10 @@ private void pullDeviceMessage() { while (true) { try { info("Waiting for swarm configuration"); - SwarmMessage.Attributes attributes = new Attributes(); + Envelope attributes = new Envelope(); Bundle pull = pubSubClient.pull(); - attributes.subFolder = pull.attributes.get("subFolder"); - if (!SWARM_SUBFOLDER.equals(attributes.subFolder)) { + attributes.subFolder = SubFolder.valueOf(pull.attributes.get("subFolder")); + if (!SubFolder.SWARM.equals(attributes.subFolder)) { error("Ignoring message with subFolder " + attributes.subFolder); continue; } @@ -373,12 +373,11 @@ private void safeSleep(int duration) { } } - private void processSwarmConfig(SwarmMessage swarm, SwarmMessage.Attributes attributes) { + private void processSwarmConfig(SwarmMessage swarm, Envelope attributes) { configuration.deviceId = Preconditions.checkNotNull(attributes.deviceId, "deviceId"); configuration.keyBytes = Base64.getDecoder() .decode(Preconditions.checkNotNull(swarm.key_base64, "key_base64")); - CloudIotConfig cloudIotConfig = makeCloudIotConfig(attributes); - configuration.endpoint = SiteModel.extractEndpointConfig(cloudIotConfig); + configuration.endpoint = SiteModel.makeEndpointConfig(attributes); processDeviceMetadata( Preconditions.checkNotNull(swarm.device_metadata, "device_metadata")); } @@ -413,15 +412,6 @@ private void processDeviceMetadata(Metadata metadata) { points.forEach((name, point) -> addPoint(makePoint(name, point))); } - private CloudIotConfig makeCloudIotConfig(Attributes attributes) { - CloudIotConfig cloudIotConfig = new CloudIotConfig(); - cloudIotConfig.registry_id = Preconditions.checkNotNull(attributes.deviceRegistryId, - "deviceRegistryId"); - cloudIotConfig.cloud_region = Preconditions.checkNotNull(attributes.deviceRegistryLocation, - "deviceRegistryLocation"); - return cloudIotConfig; - } - private synchronized void maybeRestartExecutor(int intervalMs) { if (scheduledFuture == null || intervalMs != messageDelayMs.get()) { cancelPeriodicSend(); diff --git a/pubber/src/main/java/daq/pubber/SwarmMessage.java b/pubber/src/main/java/daq/pubber/SwarmMessage.java index 7065f6dd15..58e8dede75 100644 --- a/pubber/src/main/java/daq/pubber/SwarmMessage.java +++ b/pubber/src/main/java/daq/pubber/SwarmMessage.java @@ -9,12 +9,4 @@ public class SwarmMessage { public String key_base64; public Metadata device_metadata; - - static class Attributes { - public String deviceId; - public String deviceRegistryLocation; - public String deviceRegistryId; - public String projectId; - public String subFolder; - } } diff --git a/schema/envelope.json b/schema/envelope.json index d22332d8cf..4aec304079 100644 --- a/schema/envelope.json +++ b/schema/envelope.json @@ -15,6 +15,10 @@ "type": "string", "pattern": "^[a-zA-Z][-a-zA-Z0-9._+~%]*[a-zA-Z0-9]$" }, + "deviceRegistryLocation": { + "type": "string", + "pattern": "^[a-z]+-[a-z]+1$" + }, "projectId": { "type": "string", "pattern": "^([.a-z]+:)?[a-z][-a-z0-9]*[a-z0-9]$" @@ -26,6 +30,7 @@ "discovery", "system", "gateway", + "swarm", "localnet", "pointset", "blobset" diff --git a/tests/envelope.tests/example2.json b/tests/envelope.tests/example2.json index 2cd9274e2b..7a176c5777 100644 --- a/tests/envelope.tests/example2.json +++ b/tests/envelope.tests/example2.json @@ -1,6 +1,7 @@ { "projectId": "daq-test-suite", "deviceRegistryId": "test-registry", + "deviceRegistryLocation": "us-central1", "deviceNumId": "23812", "deviceId": "FCU-002", "subFolder": "system", From 276003320d79393862cedb0837c94664b5914ccd Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Tue, 9 Aug 2022 08:08:42 -0700 Subject: [PATCH 23/27] Cleanup extras --- tests/command_mapping.tests/mapping.json | 31 ----------- tests/config_mapping.tests/mapping.json | 31 ----------- tests/event_mapping.tests/mapping.json | 32 ------------ tests/event_mapping.tests/prediction.json | 13 ----- .../staged_equip_response.txt | 52 ------------------- tests/state_mapping.tests/mapping.json | 31 ----------- 6 files changed, 190 deletions(-) delete mode 100644 tests/command_mapping.tests/mapping.json delete mode 100644 tests/config_mapping.tests/mapping.json delete mode 100644 tests/event_mapping.tests/mapping.json delete mode 100644 tests/event_mapping.tests/prediction.json delete mode 100644 tests/event_mapping.tests/staged_equip_response.txt delete mode 100644 tests/state_mapping.tests/mapping.json diff --git a/tests/command_mapping.tests/mapping.json b/tests/command_mapping.tests/mapping.json deleted file mode 100644 index 97a26464e7..0000000000 --- a/tests/command_mapping.tests/mapping.json +++ /dev/null @@ -1,31 +0,0 @@ -// -// This message represents an import of an existing device mapping into the mapper. -// -// Note that the building and device id are included as part of the message envelope. -// -{ - "version": "1.3.14", - "timestamp": "2018-08-26T21:39:29.364Z", - "guid": "21387BBA787ADD", - "translation": { - "zone_air_temperature_sensor": { - "present_value": "points.temp_1.present_value", - "units": { - "key": "pointset.points.temp_1.units", - "values": { - "degrees_celsius": "degC" - } - } - }, - "supply_air_isolation_damper_command": { - "present_value": "points.damper_1.present_value", - "states": { - "OPEN": "1", - "CLOSED": [ - "2", - "3" - ] - } - } - } -} diff --git a/tests/config_mapping.tests/mapping.json b/tests/config_mapping.tests/mapping.json deleted file mode 100644 index e2b9f7f01f..0000000000 --- a/tests/config_mapping.tests/mapping.json +++ /dev/null @@ -1,31 +0,0 @@ -// -// This message represents the entire high-level config of a recommendation model, -// as given to the mapping component. It therefore reflects the external (to a mapping engine) -// view of what has happened outside of the mapping engine itself. -// -// The building id is defined by the message envelope. -// -// General overview of the external representation: -// 1. Applied: An exported recommendation was received and applied to the system. -// 2. Request: Last time an export was requested from the system. -// -// Comaprison of timestamps would trigger action by the external (non-mapper) system: -// imported < applied: The system should import (send command) the applied mapping into the mapper -// -{ - "version": "1.3.14", - "timestamp": "2018-08-28T21:39:29.364Z", - "devices": { - "FCU-123": { - "guid": "21387BBA787ADD", - "applied": "2018-08-27T22:39:12.364Z", // Last time a mapping was successfully applied externally - "requested": "2018-08-27T22:39:12.364Z", // Timestamp of requested model export - "status": { - "message": "Update in progress", - "category": "mapping.apply.start", - "timestamp": "2018-08-28T21:39:30.364Z", - "level": 500 - } - } - } -} diff --git a/tests/event_mapping.tests/mapping.json b/tests/event_mapping.tests/mapping.json deleted file mode 100644 index d8da42a2c8..0000000000 --- a/tests/event_mapping.tests/mapping.json +++ /dev/null @@ -1,32 +0,0 @@ -// -// This message represents an output from the mapper expressing the mapping -// for a particular device. -// -// The specific building and device id would be part of the message envelope. -// -{ - "version": "1.3.14", - "timestamp": "2018-08-26T21:39:29.364Z", - "guid": "21387BBA787ADD", - "translation": { - "zone_air_temperature_sensor": { - "present_value": "points.temp_1.present_value", - "units": { - "key": "pointset.points.temp_1.units", - "values": { - "degrees_celsius": "degC" - } - } - }, - "supply_air_isolation_damper_command": { - "present_value": "points.damper_1.present_value", - "states": { - "OPEN": "1", - "CLOSED": [ - "2", - "3" - ] - } - } - } -} diff --git a/tests/event_mapping.tests/prediction.json b/tests/event_mapping.tests/prediction.json deleted file mode 100644 index 5ed99f1f28..0000000000 --- a/tests/event_mapping.tests/prediction.json +++ /dev/null @@ -1,13 +0,0 @@ -// -// This message represents a predictive model for a device. Includes all the -// probabalistic information necessary for an agent (human or otherwise) to -// specify what the actual translation should be. This is an intermediate -// state that requires 'promotion' before being properly exported. -// -{ - "version": "1.3.14", - "timestamp": "2018-08-26T21:39:29.364Z", - "prediction": { - // TODO: Include details from staged_equip_response.txt - } -} diff --git a/tests/event_mapping.tests/staged_equip_response.txt b/tests/event_mapping.tests/staged_equip_response.txt deleted file mode 100644 index d7e69fbb21..0000000000 --- a/tests/event_mapping.tests/staged_equip_response.txt +++ /dev/null @@ -1,52 +0,0 @@ -{ - "building_name": "US-MTV-1804", // building id from discovery message - "equipment": - [ - { - "data": - { - "device_id": "4b:9a:84:17:17:6c", // from UDMI discovery, ties back to a specific device - "model.equip_type_predictions": // top five predictions - { - "AHU": 0.10, - "BLR": 0.15, - "HX": 0.20, - "MAU": 0.25, - "VAV": 0.30 - }, - "entity_type": "VAV", // top prediction - "model.equip_name": "VAV-1", // predicted equipment instance ID - "guid": "8cf8c6ea-c8aa-412c-914d-947947fa51ba", // UUID V4 - "model.parent": "AHU-1", // predicted parent equipment ID - }, - "created": "2022-03-25T18:16:36+00:00", - "last_promoted": null, - "modified": null, - } - ], - "points_by_equip_name": - { - "VAV-1": - [ - { - "data": - { - "object_identifier": "('analogInupt', 1)", // from UDMI discovery - "model.point_type_predictions": // top five predictions - { - "building_air_static_pressure_sensor": 0.10, - "condensing_supply_water_temperature_sensor": 0.15, - "exhaust_fan_power_sensor": 0.20, - "discharge_fan_speed_percentage_sensor": 0.25, - "zone_air_relative_humidity_sensor": 0.30 - }, - "telemetry_field_name": "zone_air_relative_humidity_sensor", // top prediction - }, - "created": "2022-03-25T18:16:36+00:00", - "last_promoted": null, - "modified": null, - "entity.guid": "8cf8c6ea-c8aa-412c-914d-947947fa51ba", - } - ] - } -} diff --git a/tests/state_mapping.tests/mapping.json b/tests/state_mapping.tests/mapping.json deleted file mode 100644 index a4f3e54915..0000000000 --- a/tests/state_mapping.tests/mapping.json +++ /dev/null @@ -1,31 +0,0 @@ -// -// This message represents the entire high-level state of a recommendation, -// as given by the mapping component. It therefore reflects the internal state -// of the mapper as might be of interest to the outside world. -// -// Note that building id is defined as part of the message envelope. -// -// General workflow is (not all steps will always be present): -// 1. Import: Existing mappings can be imported from an external source -// 2. Discovery: Mapper receives discovery message -// 3. Prediction: Internal algorithm predicts what it might be and gives options -// 4. Promotion: Some process for promoting a single option based on prediction -// 5. Export: Result of promotion are exported for external consumption -// -// Comparison of timestamps with the related config block would trigger action by the mapper: -// requested > exported: The mapper should export (send event) the indicated mapping -// -{ - "version": "1.3.14", - "timestamp": "2018-08-26T21:39:29.364Z", - "devices": { - "FCU-123": { - "guid": "21387BBA787ADD", - "imported": "2018-08-26T21:39:28.364Z", // Last time the mapping was imported - "discovered": "2018-08-26T21:39:29.364Z", // Last received discovery message - "predicted": "2018-08-28T21:59:18.364Z", // Last recommendation staging (result of automatic prediction) - "promoted": "2018-08-28T22:39:12.364Z", // Last recommendation promotion (result of manual QA) - "exported": "2018-08-28T22:39:12.364Z", // Last time this device mapping was exported - } - } -} From b0412d193aa89a9ea06ed998209d6cefbd76b1a7 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Tue, 9 Aug 2022 08:19:12 -0700 Subject: [PATCH 24/27] Docs revert --- docs/specs/onboarding.md | 119 +++++++++++++++++++++------------------ 1 file changed, 65 insertions(+), 54 deletions(-) diff --git a/docs/specs/onboarding.md b/docs/specs/onboarding.md index d43265368a..07fa6a6ff3 100644 --- a/docs/specs/onboarding.md +++ b/docs/specs/onboarding.md @@ -3,85 +3,96 @@ # Onboarding The overall "onboarding" flow consists of a number of separate subflows stitched together for a complete -end-to-end process to take an "unknown" device and ensure that it's properly integrated with backend services. +end-to-end process. This generall starts from an "unknown" device in the system through to a UDMI-compliant +device that's properly integrated with backend services. -At a high-level, the process involves different message subgroups that handle slightly different +At a high-level, the overall process involves different message subgroups that handle slightly different scopes of device data: * **(Native)**: Device communicaiton using some non-UDMI native protocol (e.g. BACnet, Modbus, etc...) * **[Discovery](discovery.md)**: Messages relating to the discovery (and provisioning) of devices (e.g. messy BACnet info) * **[Mapping](mapping.md)**: Messages relating to a 'resolved' device type and ID (e.g. the device is an `AHU` called `AHU-1`) * **[Pointset](../messages/pointset.md)**: Messages relating to actual data flow (e.g. temperature reading), essentially the interesting stuff -* **(Onboard)**: Interactions with an external (non-UDMI) entity of some kind to facilitate onboarding of devices ## Sequence Diagram The overall onboarding sequence involves multiple components that work together to provide the overall flow: -* **Devices**: The target things that need to be discovered, configured, and ultimately communicate point data. +* **Device**: The target thing that needs to be discovered, configured, and ultimately communications point data * **Spotter**: A on-prem node responsible for handling discovery scans of devices. -* **Agent**: Agent responsible for managing the overall _discovery_ and _mapping_ process (how often, what color, etc...). -* **Engine**: Mapping engine that uses hueristics, ML, or a UI to convert discovery information into a concrete device/sink mapping. -* **Sink**: Ultimate recepient of pointset information, The thing that cares about 'temperature' in a room. +* **Cloud**: The on-prem/in-cloud boundary. Things to the left are things in the building, to the right are in the cloud +* **Agent**: Responsible for managing the overall _discovery_ process (how often, what color, etc...) +* **Mapper**: Uses hueristics, ML, or a UI to convert discovery information into a concrete device/sink mapping +* **Sink**: Ultimate recepient of pointset information. The thing that cares about 'temperature' in a room Notes & Caveats: 1. Only "interesting" messages are shown in the diagram, there's other control flow things that go on (e.g. to configure when the *Spotter* should activate) to complete the overall flow. -2. This shows the flow for a direct-connect (no IoT Gateway involved) device. The overall flow for a proxied device -(with IoT Gateway) would more or less be the same with some additional intermediaries. +2. This just shows one-of-many potential provisioning (handling keys) techniques. There's other paths +that would be possible (including manually, which is the baseline default). +3. This shows the flow for a direct-connect (no IoT Gateway) device. The overall flow for a proxied device +(with IoT Gateway) would more or less be the same, just different details about exact communication mechanisms. ``` -+---------+ +---------+ +-------+ +---------+ +-------+ -| Devices | | Spotter | | Agent | | Engine | | Sink | -+---------+ +---------+ +-------+ +---------+ +-------+ - | | | | | - | | Discovery Config | | | - | |<---------------------| | | - | | | | | - | Probes | | | | - |<-------------| | | | - | | | | | - | Replies | | | | - |------------->| | | | - | | | | | - | | Discovery Events | | | - | |------------------------------------------->| | - | | | | | - | | | Mapping Config | | - | | |-------------------->| | - | | | | | - | | | Mapping Events | | - | | |<--------------------| | - | | | | | - | | | Onboard Requests | | - | | |------------------------------->| - | | | | | - | Telemetry Events | | | - |--------------------------------------------------------------------->| - | | | | | ++---------+ +---------+ +-------+ +-------+ +--------+ +-------+ +| Device | | Spotter | | Cloud | | Agent | | Mapper | | Sink | ++---------+ +---------+ +-------+ +-------+ +--------+ +-------+ + | | | | | | + | (Info) | | | | | + |----------------------->| | | | | + | | | | | | + | | Discovery Event | | | + | |------------------------------------------------------->| | + | | | | | | + | | | | | Mapping Event | + | | | | |------------------>| + | | | | | | + | | | | Mapping Event | | + | | | |<-------------------| | + | | | | | | + | | | (Cloud Provision) | | | + | | |<----------------------| | | + | | | | | | + | | | Discovery Command | | | + | |<----------------------------------| | | + | | | | | | + | (Device Provision) | | | | | + |<-----------------------| | | | | + | | | | | | + | | | | | Pointset Config | + |<----------------------------------------------------------------------------------------------------| + | | | | | | + | Pointset Event | | | | | + |---------------------------------------------------------------------------------------------------->| + | | | | | | ``` -1. **[Discovery Config](../../tests/config.tests/discovery.json)** indicates that the _spotter_ should do the needful and scan the local network. -1. **(Probes)** scan for natively-encoded _device_ information from devices (format out of scope for UDMI) -2. **(Deplies)** contain the discovered natively-encoded _device_ information, e.g.: +1. **(Info)** contains natively-encoded _Device_ information (format out of scope for UDMI) * "I am device `78F936`, with points { `room_temp`, `step_size`, and `operation_count` }, and public key `XYZZYZ`" -2. **[Discovery Events](../../tests/event_discovery.tests/enumeration.json)** wraps the device info from the _spotter_ into a UDMI-normalized format, e.g.: +2. **[Discovery Event](../../tests/event_discovery.tests/enumeration.json)** from _Spotter_ wraps up the device info into a UDMI-normalized format * "Device `78F936` has points { }, with a public key `XYZZYZ`" -3. **[Mapping Config](../../tests/config_mapping.tests/mapping.json)** from the _agent_ indicates that the _engine_ should export responses. -3. **[Mapping Events](../../tests/event_mapping.tests/mapping.json)** from the _engine_ contain actual calculated point mappings. +3. **Mapping Event** from the _Mapper_ (recieved by both _Sink_ and _Agent_) * "Device `78F936` is an `AHU` called `AHU-183`, and `room_temp` is really a `flow_temperatue`" -4. **Onboard Requests** are requests to the appropriate _sink_ to onboard a device (contents are defined by _sink_ and out of scope for UDMI) -8. **[Telemetry Events](../../tests/event_pointset.tests/example.json)** are data events from _device_ to _sink_... business as usual! +5. **(Cloud Provision)** from _Agent_ sets up the _Cloud_ layer using the IoT Core API. + * "Device `AHU-183` exists and has public key `XYZZYZ`" +4. **[Discovery Command](../../tests/command_discovery.tests/provision.json)** from the _Agent_ to the _Spotter_ contains information necessary to provision the device + * "Device `78F936` should call itself `AHU-183` when connecting to the cloud" +5. **(Device Provision)** uses some natively-encoded mechanism for setting up the device with relevant cloud info + * "Device `78F936`, you are celled `AHU-183` when connecting to the cloud" +7. **[Pointset Config](../../tests/config.tests/example.json)** from the _Sink_ can go directly to the _Device_ (after it connects to the cloud) + * "Device `AHU-183`, you should send the `room_temp` data point every `10 minutes`" +8. **[Pointset Events](../../tests/event_pointset.tests/example.json)** sends telemetry events from the _Device_ to _Sink_... business as usual! * "I am `AHU-183`, and my `room_temp` is `73`" ## Source Created using https://textart.io/sequence# ``` -object Devices Spotter Agent Engine Sink -Agent->Spotter: Discovery Config -Spotter->Devices: Probes -Devices->Spotter: Replies -Spotter->Engine: Discovery Events -Agent->Engine: Mapping Config -Engine->Agent: Mapping Events -Agent->Sink: Onboard Requests -Devices->Sink: Telemetry Events +object Device Spotter Cloud Agent Mapper Sink +Device->Spotter: (Info) +Spotter->Mapper: Discovery Event +Mapper->Sink: Mapping Event +Mapper->Agent: Mapping Event +Agent->Cloud: (Cloud Provision) +Agent->Spotter: Discovery Command +Spotter->Device: (Device Provision) +Sink->Device: Pointset Config +Device->Sink: Pointset Event ``` From 10c08e4d9ef61063d751d5cef831fd026116a89a Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Tue, 9 Aug 2022 08:51:30 -0700 Subject: [PATCH 25/27] Fixing projectId handling --- pubber/src/main/java/daq/pubber/Configuration.java | 1 - pubber/src/main/java/daq/pubber/Pubber.java | 6 ++++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pubber/src/main/java/daq/pubber/Configuration.java b/pubber/src/main/java/daq/pubber/Configuration.java index 7cf8d206db..feae4bd908 100644 --- a/pubber/src/main/java/daq/pubber/Configuration.java +++ b/pubber/src/main/java/daq/pubber/Configuration.java @@ -6,7 +6,6 @@ */ public class Configuration { public EndpointConfiguration endpoint = new EndpointConfiguration(); - public String projectId; public String gatewayId; public String deviceId; public String sitePath; diff --git a/pubber/src/main/java/daq/pubber/Pubber.java b/pubber/src/main/java/daq/pubber/Pubber.java index fa61cb6993..5ab9167b97 100644 --- a/pubber/src/main/java/daq/pubber/Pubber.java +++ b/pubber/src/main/java/daq/pubber/Pubber.java @@ -128,6 +128,7 @@ public class Pubber { private final ExtraPointsetEvent devicePoints = new ExtraPointsetEvent(); private final Set allPoints = new HashSet<>(); private final AtomicBoolean stateDirty = new AtomicBoolean(); + private final String projectId; private Config deviceConfig = new Config(); private int deviceMessageCount = -1; private MqttPublisher mqttPublisher; @@ -149,6 +150,7 @@ public Pubber(String configPath) { File configFile = new File(configPath); try { configuration = OBJECT_MAPPER.readValue(configFile, Configuration.class); + projectId = configuration.endpoint.projectId; } catch (UnrecognizedPropertyException e) { throw new RuntimeException("Invalid arguments or options: " + e.getMessage()); } catch (Exception e) { @@ -165,8 +167,8 @@ public Pubber(String configPath) { * @param serialNo Serial number of the device */ public Pubber(String projectId, String sitePath, String deviceId, String serialNo) { + this.projectId = projectId; configuration = new Configuration(); - configuration.projectId = projectId; configuration.deviceId = deviceId; configuration.serialNo = serialNo; if (PUBSUB_SITE.equals(sitePath)) { @@ -293,7 +295,7 @@ private void initializeDevice() { if (configuration.sitePath != null) { siteModel = new SiteModel(configuration.sitePath); - siteModel.initialize(configuration.projectId); + siteModel.initialize(projectId); configuration.endpoint = siteModel.getEndpointConfig(); processDeviceMetadata(siteModel.getMetadata(configuration.deviceId)); } else if (pubSubClient != null) { From f757f63bd9d5c36f2ba360e65cb15d0469bae7ae Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Tue, 9 Aug 2022 10:41:07 -0700 Subject: [PATCH 26/27] Moving endpoint configuration --- .../main/java/com/google/udmi/util}/EndpointConfiguration.java | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {pubber/src/main/java/daq/pubber => common/src/main/java/com/google/udmi/util}/EndpointConfiguration.java (100%) diff --git a/pubber/src/main/java/daq/pubber/EndpointConfiguration.java b/common/src/main/java/com/google/udmi/util/EndpointConfiguration.java similarity index 100% rename from pubber/src/main/java/daq/pubber/EndpointConfiguration.java rename to common/src/main/java/com/google/udmi/util/EndpointConfiguration.java From 48c5ba5322ca0c8b1faee36f14d2f78e18ba9528 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Tue, 9 Aug 2022 10:43:20 -0700 Subject: [PATCH 27/27] Clean up package and imports --- .../main/java/com/google/udmi/util/EndpointConfiguration.java | 2 +- common/src/main/java/com/google/udmi/util/SiteModel.java | 1 - pubber/src/main/java/daq/pubber/Configuration.java | 2 ++ pubber/src/main/java/daq/pubber/Pubber.java | 1 + 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/common/src/main/java/com/google/udmi/util/EndpointConfiguration.java b/common/src/main/java/com/google/udmi/util/EndpointConfiguration.java index 947e5b0d5d..006d572bf8 100644 --- a/common/src/main/java/com/google/udmi/util/EndpointConfiguration.java +++ b/common/src/main/java/com/google/udmi/util/EndpointConfiguration.java @@ -1,4 +1,4 @@ -package daq.pubber; +package com.google.udmi.util; /** * Configuration for the connection endpoint. diff --git a/common/src/main/java/com/google/udmi/util/SiteModel.java b/common/src/main/java/com/google/udmi/util/SiteModel.java index 1033e2a187..51c9434d3c 100644 --- a/common/src/main/java/com/google/udmi/util/SiteModel.java +++ b/common/src/main/java/com/google/udmi/util/SiteModel.java @@ -7,7 +7,6 @@ import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.databind.util.ISO8601DateFormat; import com.google.common.base.Preconditions; -import daq.pubber.EndpointConfiguration; import java.io.File; import java.util.Arrays; import java.util.Map; diff --git a/pubber/src/main/java/daq/pubber/Configuration.java b/pubber/src/main/java/daq/pubber/Configuration.java index feae4bd908..3c3778d34c 100644 --- a/pubber/src/main/java/daq/pubber/Configuration.java +++ b/pubber/src/main/java/daq/pubber/Configuration.java @@ -1,6 +1,8 @@ package daq.pubber; +import com.google.udmi.util.EndpointConfiguration; + /** * General bucket of pubber configuration information. */ diff --git a/pubber/src/main/java/daq/pubber/Pubber.java b/pubber/src/main/java/daq/pubber/Pubber.java index 5ab9167b97..a1f245e10b 100644 --- a/pubber/src/main/java/daq/pubber/Pubber.java +++ b/pubber/src/main/java/daq/pubber/Pubber.java @@ -11,6 +11,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.daq.mqtt.util.CatchingScheduledThreadPoolExecutor; +import com.google.udmi.util.EndpointConfiguration; import com.google.udmi.util.SiteModel; import daq.pubber.MqttPublisher.PublisherException; import daq.pubber.PubSubClient.Bundle;