Add additional gcloud miscellaneous tooling (#491)

misc/setup_gcp_project → misc/gcloud_deployment/setup_gcp_project

@@ -28,7 +28,7 @@ gcloud iam service-accounts create $SERVICE_ACCOUNT_NAME --display-name="UDMI Cl
 gcloud projects add-iam-policy-binding $PROJECT_ID --member="serviceAccount:$SERVICE_ACCOUNT" --role="roles/pubsub.publisher" 
 gcloud projects add-iam-policy-binding $PROJECT_ID --member="serviceAccount:$SERVICE_ACCOUNT" --role="roles/cloudiot.provisioner" 
 
-$ROOT_DIR/dashboard/deploy_dashboard_gcloud $PROJECT_ID --service-account=$SERVICE_ACCOUNT
+$ROOT_DIR/udmis/deploy_udmis_gcloud $PROJECT_ID --service-account=$SERVICE_ACCOUNT
 
 $ROOT_DIR/bin/clone_model
 registry_id=$(jq -r .registry_id $ROOT_DIR/sites/udmi_site_model/cloud_iot_config.json)

misc/gcloud_iot_config/README.md

@@ -0,0 +1,6 @@
+# GCloud IoT Config Store
+
+Stores a complete history of all config messages sent to devices in a GCP Project in GCS
+
+## Installation
+`./deploy PROJECT_ID`

misc/gcloud_iot_config/deploy

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+if (( $# < 1 )); then
+    echo $0 PROJECT_ID 
+    exit
+fi
+
+PROJECT_ID=$1
+shift 1
+
+FUNCTION_NAME=gcloud_iot_config
+ENTRY_POINT=storeConfig
+TOPIC=config_updates
+BUCKET=$(PROJECT_ID)-iot-configs
+
+## storage buckets create gs://$BUCKET_NAME --project=$PROJECT_ID --default-storage-class=STANDARD --location=us-central1 
+gcloud storage buckets create gs://$BUCKET --project=$PROJECT_ID --default-storage-class=STANDARD --location=us-central1 
+
+# Create pub/sub topic
+gcloud pubsub topics create $TOPIC --project=$PROJECT_ID
+
+# create logs router
+gcloud logging sinks create config_updates_logsink \
+    pubsub.googleapis.com/projects/$PROJECT_ID/topics/$TOPIC \
+    --log-filter='protoPayload.methodName="google.cloud.iot.v1.DeviceManager.ModifyCloudToDeviceConfig" AND severity=NOTICE' \
+    --project=$PROJECT_ID
+
+LOGSINK_SERVICE_ACCOUNT=$(gcloud logging sinks describe config_updates_logsink --project=$PROJECT_ID | grep writerIdentity | sed "s/writerIdentity: //")
+gcloud pubsub topics add-iam-policy-binding $TOPIC --project=$PROJECT_ID --member=$LOGSINK_SERVICE_ACCOUNT --role='roles/pubsub.publisher'
+
+# Deploy Cloud Function
+gcloud functions deploy $FUNCTION_NAME \
+    --trigger-topic=$TOPIC\
+    --entry-point=$ENTRY_POINT \
+    --runtime nodejs16 \
+    --project=$PROJECT_ID \
+    --source=functions/ \
+    --set-env-vars PROJECT_ID=$PROJECT_ID,DATASET_ID=$DATASET_ID,BUCKET=$BUCKET

misc/gcloud_iot_config/functions/index.js

@@ -0,0 +1,58 @@
+const PROJECT_ID = process.env.PROJECT_ID
+const DATASET_ID = process.env.DATASET_ID
+const BUCKET = process.env.BUCKET
+
+const iot = require('@google-cloud/iot');
+const {Storage} = require('@google-cloud/storage');
+const storage = new Storage();
+const iotClient = new iot.v1.DeviceManagerClient({});
+const bucket = storage.bucket(BUCKET);
+var fs = require('fs');
+
+// projects/daq1-273309/locations/us-central1/registries/ZZ-TRI-FECTA/devices/AHU-1 
+
+exports.storeConfig = async (event, context) => {
+  const pubsubMessage = event.data;
+  const objStr = Buffer.from(pubsubMessage, 'base64').toString();
+  const logData = JSON.parse(objStr);
+
+  devicePath = logData['protoPayload']['resourceName']
+  versionToUpdate = logData['protoPayload']['request']['versionToUpdate']
+
+  const [response] = await iotClient.listDeviceConfigVersions({
+    name: devicePath,
+  });
+
+  splitDevice = devicePath.split('/')
+  registryId = splitDevice[5]
+  deviceId = splitDevice[7]
+
+  const configs = response.deviceConfigs;
+
+  if (configs.length === 0) {
+    return null;
+  }
+
+  configs.forEach((config, index) => { 
+    if(config.version == versionToUpdate){
+      configPayload = config.binaryData.toString('utf8');
+      localFileName = `/tmp/${registryId}_${deviceId}_${versionToUpdate}.txt`
+
+      fs.writeFile(localFileName, configPayload, function (err) {
+        console.log(err)
+      });
+
+
+      const options = {
+        destination: `${registryId}/${deviceId}/${versionToUpdate}.json`
+      };
+
+      bucket.upload(localFileName, options, function(err, file) {
+        console.log(err)
+      });
+
+    }
+  });
+
+};
+

misc/gcloud_iot_config/functions/package.json

@@ -0,0 +1,9 @@
+{
+    "name": "config_to_gcs",
+    "version": "0.0.1",
+    "dependencies": {
+      "@google-cloud/pubsub": "^0.18.0",
+      "@google-cloud/iot": "2.3.4",
+      "@google-cloud/storage": "6.5.2"
+    } 
+  }

misc/gcloud_iot_connection_log/README.md

@@ -0,0 +1,47 @@
+# GCloud IoT Connection Logs
+
+Stores all device connections and disconnection from IoT Core into BigQuery
+
+**NOTE**
+- Requires **Cloud Logging** on devices or registry be set to `INFO` or more.
+- IoT Core has a default log entries limit of 2000 per second. If a registry has `DEBUG` level logging, this may very quickly be exceeding, and will result in missing connection or disconnection log events
+
+## Installation
+
+`./deploy PROJECT_ID DATASET_ID LOCATION TRIGGER_TOPIC [--drop]`
+
+**NOTE** cloud function deployed with default app-engine service account. This may need to change if this account does not have required permissions
+
+## Example Queries
+
+### List devices which were once connected that now offline 
+
+```sql
+ SELECT   *,
+         Row_number() OVER(partition BY device_id, registry_id ORDER BY timestamp DESC) AS rank
+FROM     `PROJECT_ID.udmi.iot_connects` qualify rank = 1
+AND      event = 0
+ORDER BY timestamp DESC
+```
+
+### List device outages exceeding X minutes
+
+```sql 
+SELECT device_id,
+       qtimestamp disconnect_time,
+       timestamp  reconnect_time,
+       outage_minutes
+FROM   (
+              SELECT device_id,
+                     qtimestamp,
+                     timestamp,
+                     Datetime_diff(timestamp, qtimestamp, minute) AS outage_minutes
+              FROM   (
+                              SELECT   *,
+                                       Lag(timestamp) OVER (partition BY device_id, registry_id ORDER BY timestamp, event) qtimestamp
+                              FROM     `PROJECT_ID.udmi.iot_connects`
+                              WHERE    logentry = 1
+                              AND      event = 1 )
+              WHERE  timestamp IS NOT NULL )
+WHERE  outage_minutes > 10
+```

misc/gcloud_iot_connection_log/deploy

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+if (( $# < 4 )); then
+    echo $0 PROJECT_ID DATASET_ID LOCATION TRIGGER_TOPIC [--drop]
+    exit
+fi
+
+PROJECT_ID=$1
+DATASET_ID=$2
+LOCATION=$3
+TRIGGER_TOPIC=$4
+shift 4
+
+
+
+if [[ $1 == "--drop" ]]; then
+    shift 1
+    #echo "WARNING: Dropping tables in 5 seconds. Data will be permanently lost"
+    #sleep 5 && echo "Deleting tables ..." && sleep 3
+    bq rm -t -f $PROJECT_ID:$DATASET_ID.iot_connects
+fi
+
+FUNCTION_NAME=gcloud_iot_connection_log
+TOPIC=iot_connections
+
+# State table
+bq mk \
+    --table \
+    $PROJECT_ID:$DATASET_ID.iot_connects \
+    schema_iot_logs.json 
+
+# Create pub/sub topic
+gcloud pubsub topics create $TOPIC
+
+# create logs router
+gcloud logging sinks create iot_connections_logsink \
+    pubsub.googleapis.com/projects/$PROJECT_ID/topics/$TOPIC \
+    --log-filter='resource.type="gce_instance"'
+
+LOGSINK_SERVICE_ACCOUNT=$(gcloud logging sinks describe iot_connections_logsink --project=$PROJECT_ID | grep writerIdentity | sed "s/writerIdentity: //")
+gcloud pubsub topics add-iam-policy-binding $TOPIC --project=$PROJECT_ID --member=$LOGSINK_SERVICE_ACCOUNT --role='roles/pubsub.publisher'
+
+# Deploy Cloud Function
+gcloud functions deploy $FUNCTION_NAME \
+    --trigger-topic=logtest\
+    --entry-point=logConnectionEvents \
+    --runtime nodejs16 \
+    --project=$PROJECT_ID \
+    --source=functions/ \
+    --set-env-vars PROJECT_ID=$PROJECT_ID,DATASET_ID=$DATASET_ID

misc/gcloud_iot_connection_log/functions/index.js

@@ -0,0 +1,76 @@
+const {BigQuery} = require('@google-cloud/bigquery');
+const bigquery = new BigQuery();
+
+const PROJECT_ID = process.env.PROJECT_ID
+const DATASET_ID = process.env.DATASET_ID
+
+exports.logConnectionEvents = async (event, context) => {
+  const pubsubMessage = event.data;
+  const objStr = Buffer.from(pubsubMessage, 'base64').toString();
+  const logData = JSON.parse(objStr);
+
+  if (logData['jsonPayload']['eventType'] == "CONNECT"){
+    oldState = 0;
+    newState = 1;
+  } else if (logData['jsonPayload']['eventType']  == "DISCONNECT") {
+    newState = 0;
+    oldState = 1;
+  } else {
+    return;
+  }
+
+  if (logData['jsonPayload']['status']['description'] == "OK" ){
+    description = null;
+    message = null;
+  } else {
+    description = logData['jsonPayload']['status']['description'];
+    message = logData['jsonPayload']['status']['message'];
+  }
+
+  // reduce nanoseconds to microseconds
+  ts_ts = logData['timestamp'].substring(0,19)
+  ts_ns = logData['timestamp'].substring(20,29)
+  ms = Math.floor(parseInt(ts_ns) / 1000).padStart(6, '0')
+  ts = ts_ts + '.' + String(ms).padStart(6, '0') + 'Z';
+
+  // Order events by timestamp
+  // We don't need microsecond accuracy, so either add or subtract 1 microsecond
+  // to avoid dealing with dates and set such that t1 < t2
+  if (ms == 0) {
+    t1 = ts
+    t2 = ts_ts + '.000001Z';
+  } else {
+    t2 = ts;
+    t1 = ts_ts + '.' + String(ms - 1).padStart(6, '0') + 'Z';
+  }
+
+
+// the log event is always the newer entry
+log_entry = {
+    timestamp: t2,
+    device_id: logData['labels']['device_id'],
+    device_num_id: logData['resource']['labels']['device_num_id'],
+    registry_id: logData['resource']['labels']['device_registry_id'],
+    event: newState,
+    logentry: 1,
+    logentry_description: description,
+    logentry_message: message
+  }
+
+
+  // the "derivative" is always preceding entry 
+  log_derivative = {
+    timestamp: t1,
+    device_id: logData['labels']['device_id'],
+    device_num_id: logData['resource']['labels']['device_num_id'],
+    registry_id: logData['resource']['labels']['device_registry_id'],
+    event: oldState,
+    logentry: 0
+  }
+
+  return await Promise.all([
+    bigquery.dataset(DATASET_ID).table('iot_connects').insert([log_derivative, log_entry]),
+  ]);
+
+};
+

misc/gcloud_iot_connection_log/functions/package.json

@@ -0,0 +1,8 @@
+{
+    "name": "iotlogs-to-bq",
+    "version": "0.0.1",
+    "dependencies": {
+      "@google-cloud/pubsub": "^0.18.0",
+      "@google-cloud/bigquery": "^3.0.0"
+    } 
+  }

misc/gcloud_iot_connection_log/schema_iot_logs.json

@@ -0,0 +1,42 @@
+[
+  {
+    "mode": "NULLABLE",
+    "name": "timestamp",
+    "type": "TIMESTAMP"
+    },
+    {
+    "mode": "NULLABLE",
+    "name": "registry_id",
+    "type": "STRING"
+    },
+    {
+    "mode": "NULLABLE",
+    "name": "device_id",
+    "type": "STRING"
+    },
+    {
+      "mode": "NULLABLE",
+      "name": "device_num_id",
+      "type": "STRING"
+    },
+    {
+      "mode": "NULLABLE",
+      "name": "event",
+      "type": "INTEGER"
+    },
+    {
+      "mode": "NULLABLE",
+      "name": "logentry",
+      "type": "INTEGER"
+    },
+    {
+      "mode": "NULLABLE",
+      "name": "logentry_description",
+      "type": "STRING"
+    },
+    {
+      "mode": "NULLABLE",
+      "name": "logentry_message",
+      "type": "STRING"
+    }
+]

misc/gcloud_messages_telemetry/README.md

@@ -0,0 +1,14 @@
+# GCloud Messages and Telemetry to BigQuery
+
+Saves a record of all messages into BigQuery, aswell as telemetry into a narrow data structure
+
+## Installation
+
+`./deploy PROJECT_ID DATASET_ID LOCATION TRIGGER_TOPIC [--drop]`
+
+**NOTE** cloud function deployed with default app-engine service account. This may need to change if this account does not have required permissions
+
+## BigQuery Tables
+
+- `messages` record of messages
+- `telemetry` telemetry