various fixes to scripts in misc folder

misc/gcloud_deployment/setup_gcp_project

@@ -1,4 +1,4 @@
-#!/bin/bash -e
+#!/bin/bash 
 # Setups a blank GCP project with the UDMI cloud functions and the registers
 # the default site model
 #
@@ -12,7 +12,7 @@ PROJECT_ID=$1
 shift 1
 
 SERVICE_ACCOUNT_NAME=udmi-cloud-functions
-SERVICE_ACCOUNT=$(SERVICE_ACCOUNT_NAME)@$(PROJECT_ID).iam.gserviceaccount.com 
+SERVICE_ACCOUNT=${SERVICE_ACCOUNT_NAME}@${PROJECT_ID}.iam.gserviceaccount.com 
 
 ROOT_DIR=$(git rev-parse --show-toplevel)
 
@@ -37,10 +37,11 @@ cloud_region=$(jq -r .cloud_region $ROOT_DIR/sites/udmi_site_model/cloud_iot_con
 gcloud iot registries create $registry_id \
     --project=$PROJECT_ID \
     --region=$cloud_region \
+    --log-level=INFO \
     --event-notification-config=topic=projects/$PROJECT_ID/topics/udmi_target \
     --state-pubsub-topic=projects/$PROJECT_ID/topics/udmi_state
 
-$ROOT_DIR/bin/registrar sites/udmi_site_model $PROJECT_ID
+$ROOT_DIR/bin/registrar $ROOT_DIR/sites/udmi_site_model $PROJECT_ID
 
 gcloud iot registries create UDMS-REFLECT \
     --project=$PROJECT_ID \

misc/gcloud_iot_config/deploy

@@ -11,7 +11,7 @@ shift 1
 FUNCTION_NAME=gcloud_iot_config
 ENTRY_POINT=storeConfig
 TOPIC=config_updates
-BUCKET=$(PROJECT_ID)-iot-configs
+BUCKET=${PROJECT_ID}-iot-configs
 
 ## storage buckets create gs://$BUCKET_NAME --project=$PROJECT_ID --default-storage-class=STANDARD --location=us-central1 
 gcloud storage buckets create gs://$BUCKET --project=$PROJECT_ID --default-storage-class=STANDARD --location=us-central1 

misc/gcloud_iot_connection_log/deploy

@@ -1,50 +1,53 @@
 #!/bin/bash
 
-if (( $# < 4 )); then
-    echo $0 PROJECT_ID DATASET_ID LOCATION TRIGGER_TOPIC [--drop]
+if (( $# < 3 )); then
+    echo $0 PROJECT_ID DATASET_ID LOCATION [--drop]
     exit
 fi
 
 PROJECT_ID=$1
 DATASET_ID=$2
 LOCATION=$3
-TRIGGER_TOPIC=$4
-shift 4
-
+shift 3
 
+FUNCTION_NAME=gcloud_iot_connection_log
+TOPIC=iot_connection_log
+TABLE_NAME=iot_connection_log
 
 if [[ $1 == "--drop" ]]; then
     shift 1
-    #echo "WARNING: Dropping tables in 5 seconds. Data will be permanently lost"
-    #sleep 5 && echo "Deleting tables ..." && sleep 3
-    bq rm -t -f $PROJECT_ID:$DATASET_ID.iot_connects
+    echo "WARNING: Dropping tables in 5 seconds. Data will be permanently lost"
+    sleep 3 && echo "Deleting tables ..." && sleep 3
+    echo bq rm -t -f $PROJECT_ID:$DATASET_ID.$TABLE_NAME
+    bq rm -t -f $PROJECT_ID:$DATASET_ID.$TABLE_NAME
 fi
 
-FUNCTION_NAME=gcloud_iot_connection_log
-TOPIC=iot_connections
-
 # State table
 bq mk \
     --table \
-    $PROJECT_ID:$DATASET_ID.iot_connects \
+    --time_partitioning_field timestamp \
+    --time_partitioning_type DAY \
+    --clustering_fields device_registry_id,device_id \
+    $PROJECT_ID:$DATASET_ID.$TABLE_NAME \
     schema_iot_logs.json 
 
 # Create pub/sub topic
-gcloud pubsub topics create $TOPIC
+gcloud pubsub topics create $TOPIC --project=$PROJECT_ID 
 
 # create logs router
 gcloud logging sinks create iot_connections_logsink \
     pubsub.googleapis.com/projects/$PROJECT_ID/topics/$TOPIC \
-    --log-filter='resource.type="gce_instance"'
+    --log-filter='resource.type="cloudiot_device" AND (jsonPayload.eventType="DISCONNECT" OR jsonPayload.eventType="CONNECT")' \
+    --project=$PROJECT_ID
 
 LOGSINK_SERVICE_ACCOUNT=$(gcloud logging sinks describe iot_connections_logsink --project=$PROJECT_ID | grep writerIdentity | sed "s/writerIdentity: //")
-gcloud pubsub topics add-iam-policy-binding $TOPIC --project=$PROJECT_ID --member=$LOGSINK_SERVICE_ACCOUNT --role='roles/pubsub.publisher'
+gcloud pubsub topics add-iam-policy-binding $TOPIC --member=$LOGSINK_SERVICE_ACCOUNT --role='roles/pubsub.publisher' --project=$PROJECT_ID 
 
 # Deploy Cloud Function
 gcloud functions deploy $FUNCTION_NAME \
-    --trigger-topic=logtest\
+    --trigger-topic=$TOPIC\
     --entry-point=logConnectionEvents \
     --runtime nodejs16 \
     --project=$PROJECT_ID \
     --source=functions/ \
-    --set-env-vars PROJECT_ID=$PROJECT_ID,DATASET_ID=$DATASET_ID
+    --set-env-vars PROJECT_ID=$PROJECT_ID,DATASET_ID=$DATASET_ID,TABLE_NAME=$TABLE_NAME

misc/gcloud_iot_connection_log/functions/index.js

@@ -1,8 +1,10 @@
 const {BigQuery} = require('@google-cloud/bigquery');
 const bigquery = new BigQuery();
+const util = require('util');
 
 const PROJECT_ID = process.env.PROJECT_ID
 const DATASET_ID = process.env.DATASET_ID
+const TABLE_NAME = process.env.TABLE_NAME
 
 exports.logConnectionEvents = async (event, context) => {
   const pubsubMessage = event.data;
@@ -30,7 +32,7 @@ exports.logConnectionEvents = async (event, context) => {
   // reduce nanoseconds to microseconds
   ts_ts = logData['timestamp'].substring(0,19)
   ts_ns = logData['timestamp'].substring(20,29)
-  ms = Math.floor(parseInt(ts_ns) / 1000).padStart(6, '0')
+  ms = Math.floor(parseInt(ts_ns) / 1000)
   ts = ts_ts + '.' + String(ms).padStart(6, '0') + 'Z';
 
   // Order events by timestamp
@@ -50,7 +52,7 @@ log_entry = {
     timestamp: t2,
     device_id: logData['labels']['device_id'],
     device_num_id: logData['resource']['labels']['device_num_id'],
-    registry_id: logData['resource']['labels']['device_registry_id'],
+    device_registry_id: logData['resource']['labels']['device_registry_id'],
     event: newState,
     logentry: 1,
     logentry_description: description,
@@ -63,14 +65,13 @@ log_entry = {
     timestamp: t1,
     device_id: logData['labels']['device_id'],
     device_num_id: logData['resource']['labels']['device_num_id'],
-    registry_id: logData['resource']['labels']['device_registry_id'],
+    device_registry_id: logData['resource']['labels']['device_registry_id'],
     event: oldState,
     logentry: 0
   }
 
   return await Promise.all([
-    bigquery.dataset(DATASET_ID).table('iot_connects').insert([log_derivative, log_entry]),
+    bigquery.dataset(DATASET_ID).table(TABLE_NAME).insert([log_derivative, log_entry]),
   ]);
-
 };
 

misc/gcloud_iot_connection_log/schema_iot_logs.json

@@ -6,7 +6,7 @@
     },
     {
     "mode": "NULLABLE",
-    "name": "registry_id",
+    "name": "device_registry_id",
     "type": "STRING"
     },
     {

misc/gcloud_iot_pingreq_log/README.md

@@ -0,0 +1,47 @@
+# GCloud IoT Connection Logs
+
+Stores all device connections and disconnection from IoT Core into BigQuery
+
+**NOTE**
+- Requires **Cloud Logging** on devices or registry be set to `INFO` or more.
+- IoT Core has a default log entries limit of 2000 per second. If a registry has `DEBUG` level logging, this may very quickly be exceeding, and will result in missing connection or disconnection log events
+
+## Installation
+
+`./deploy PROJECT_ID DATASET_ID LOCATION TRIGGER_TOPIC [--drop]`
+
+**NOTE** cloud function deployed with default app-engine service account. This may need to change if this account does not have required permissions
+
+## Example Queries
+
+### List devices which were once connected that now offline 
+
+```sql
+ SELECT   *,
+         Row_number() OVER(partition BY device_id, registry_id ORDER BY timestamp DESC) AS rank
+FROM     `PROJECT_ID.udmi.iot_connects` qualify rank = 1
+AND      event = 0
+ORDER BY timestamp DESC
+```
+
+### List device outages exceeding X minutes
+
+```sql 
+SELECT device_id,
+       qtimestamp disconnect_time,
+       timestamp  reconnect_time,
+       outage_minutes
+FROM   (
+              SELECT device_id,
+                     qtimestamp,
+                     timestamp,
+                     Datetime_diff(timestamp, qtimestamp, minute) AS outage_minutes
+              FROM   (
+                              SELECT   *,
+                                       Lag(timestamp) OVER (partition BY device_id, registry_id ORDER BY timestamp, event) qtimestamp
+                              FROM     `PROJECT_ID.udmi.iot_connects`
+                              WHERE    logentry = 1
+                              AND      event = 1 )
+              WHERE  timestamp IS NOT NULL )
+WHERE  outage_minutes > 10
+```

misc/gcloud_iot_pingreq_log/deploy

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+if (( $# < 3 )); then
+    echo $0 PROJECT_ID DATASET_ID LOCATION [--drop]
+    exit
+fi
+
+PROJECT_ID=$1
+DATASET_ID=$2
+LOCATION=$3
+shift 3
+
+FUNCTION_NAME=gcloud_iot_pingreq_log
+TOPIC=iot_pingreq_log
+TABLE_NAME=bos_messages
+LOGSINK_NAME=iot_pingreq_log
+
+# State table
+bq mk \
+    --table \
+    --time_partitioning_field publish_timestamp \
+    --time_partitioning_type DAY \
+    --clustering_fields device_registry_id,device_id \
+    $PROJECT_ID:$DATASET_ID.$TABLE_NAME \
+    schema_messages.json 
+
+# Create pub/sub topic
+gcloud pubsub topics create $TOPIC --project=$PROJECT_ID 
+
+# create logs router
+gcloud logging sinks create $LOGSINK_NAME \
+    pubsub.googleapis.com/projects/$PROJECT_ID/topics/$TOPIC \
+    --log-filter='resource.type="cloudiot_device" AND jsonPayload.eventType="PINGREQ"' \
+    --project=$PROJECT_ID
+
+LOGSINK_SERVICE_ACCOUNT=$(gcloud logging sinks describe $LOGSINK_NAME --project=$PROJECT_ID | grep writerIdentity | sed "s/writerIdentity: //")
+gcloud pubsub topics add-iam-policy-binding $TOPIC --member=$LOGSINK_SERVICE_ACCOUNT --role='roles/pubsub.publisher' --project=$PROJECT_ID 
+
+# Deploy Cloud Function
+gcloud functions deploy $FUNCTION_NAME \
+    --trigger-topic=$TOPIC\
+    --entry-point=logPingReq \
+    --runtime nodejs16 \
+    --project=$PROJECT_ID \
+    --source=functions/ \
+    --set-env-vars PROJECT_ID=$PROJECT_ID,DATASET_ID=$DATASET_ID,TABLE_NAME=$TABLE_NAME

misc/gcloud_iot_pingreq_log/functions/index.js

@@ -0,0 +1,33 @@
+const {BigQuery} = require('@google-cloud/bigquery');
+const bigquery = new BigQuery();
+
+const PROJECT_ID = process.env.PROJECT_ID
+const DATASET_ID = process.env.DATASET_ID
+const TABLE_NAME = process.env.TABLE_NAME
+
+const MESSAGE_TYPE_PINGREQ = 8;
+
+exports.logPingReq = async (event, context) => {
+  const pubsubMessage = event.data;
+  const objStr = Buffer.from(pubsubMessage, 'base64').toString();
+  const logData = JSON.parse(objStr);
+
+  if (logData['jsonPayload']['eventType'] != "PINGREQ"){
+   return;
+  }
+
+  log_entry = {
+    publish_timestamp: logData['timestamp'],
+    device_id: logData['labels']['device_id'],
+    device_num_id: logData['resource']['labels']['device_num_id'],
+    device_registry_id: logData['resource']['labels']['device_registry_id'],
+    message_type: MESSAGE_TYPE_PINGREQ,
+    payload_size_bytes: 0
+  }
+
+  return await Promise.all([
+    bigquery.dataset(DATASET_ID).table(TABLE_NAME).insert([log_entry]),
+  ]);
+
+};
+

misc/gcloud_iot_pingreq_log/functions/package.json

@@ -0,0 +1,8 @@
+{
+    "name": "iotlogs-to-bq",
+    "version": "0.0.1",
+    "dependencies": {
+      "@google-cloud/pubsub": "^0.18.0",
+      "@google-cloud/bigquery": "^3.0.0"
+    } 
+  }

misc/gcloud_iot_pingreq_log/schema_iot_logs.json

@@ -0,0 +1,42 @@
+[
+  {
+    "mode": "NULLABLE",
+    "name": "timestamp",
+    "type": "TIMESTAMP"
+    },
+    {
+    "mode": "NULLABLE",
+    "name": "device_registry_id",
+    "type": "STRING"
+    },
+    {
+    "mode": "NULLABLE",
+    "name": "device_id",
+    "type": "STRING"
+    },
+    {
+      "mode": "NULLABLE",
+      "name": "device_num_id",
+      "type": "STRING"
+    },
+    {
+      "mode": "NULLABLE",
+      "name": "event",
+      "type": "INTEGER"
+    },
+    {
+      "mode": "NULLABLE",
+      "name": "logentry",
+      "type": "INTEGER"
+    },
+    {
+      "mode": "NULLABLE",
+      "name": "logentry_description",
+      "type": "STRING"
+    },
+    {
+      "mode": "NULLABLE",
+      "name": "logentry_message",
+      "type": "STRING"
+    }
+]

misc/gcloud_iot_pingreq_log/schema_messages.json

@@ -0,0 +1,42 @@
+[
+  {
+    "mode": "NULLABLE",
+    "name": "publish_timestamp",
+    "type": "TIMESTAMP"
+  },
+  {
+    "mode": "NULLABLE",
+    "name": "device_registry_id",
+    "type": "STRING"
+  },
+  {
+    "mode": "NULLABLE",
+    "name": "device_num_id",
+    "type": "INTEGER"
+  },
+  {
+    "mode": "NULLABLE",
+    "name": "device_id",
+    "type": "STRING"
+  },
+  {
+    "mode": "NULLABLE",
+    "name": "gateway_id",
+    "type": "STRING"
+  },
+  {
+    "mode": "NULLABLE",
+    "name": "message_type",
+    "type": "INTEGER"
+  },
+  {
+    "mode": "NULLABLE",
+    "name": "message_id",
+    "type": "INTEGER"
+  },
+  {
+    "mode": "NULLABLE",
+    "name": "payload_size_bytes",
+    "type": "INTEGER"
+  }
+]