Resolve single cell geo locations

.github/workflows/test-and-release.yaml

@@ -82,9 +82,21 @@ jobs:
       - name: Run End-to-End Tests
         run: npm run test:e2e
 
+      - name: Print failed End-to-End tests
+        if: failure()
+        run:
+          cat e2e-test-result.json | npx tsx --no-warnings
+          ./feature-runner/console-reporter.ts  --only-failed --with-timestamps
+
+      - uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: e2e-test-result
+          path: e2e-test-result.json
+
       - name: Get logs
         if: failure()
-        run: ./cli.sh logs
+        run: ./cli.sh logs -f ERROR
 
       - name: Save stack outputs
         id: stack_outputs

.gitignore

@@ -9,4 +9,5 @@ cdk.out/
 dist/
 .envrc
 /certificates/
-cdk.context.json
+cdk.context.json
+e2e-test-result.json

README.md

@@ -36,6 +36,13 @@ need to prepare nRF Cloud API key.
 ./cli.sh create-health-check-device
 ```
 
+#### nRF Cloud Location Services Service Key
+
+The single-cell geo-location features uses the nRF Cloud
+[Ground Fix API](https://api.nrfcloud.com/v1#tag/Ground-Fix) which requires the
+service to be enabled in the account's plan. Manage the account at
+<https://nrfcloud.com/#/manage-plan>.
+
 ### Deploy
 
 ```bash

adr/004-resolve-all-cell-geo-locations.md

@@ -0,0 +1,20 @@
+# 004: Resolve all cell geo locations
+
+All (single) cell geo locations are resolved as soon as a device sends new
+network information instead of resolving it only on user request, or if a
+websocket connection is active for the device (meaning a user is observing the
+device on the web application).
+
+This is in line with the ground fix implementation: all ground fix messages by
+devices are resolved.
+
+Resolving all device locations based on the device's network information allows
+to:
+
+1. show device location on the map immediately (if it is already resolved)
+2. show an approximate location right after the device has connected (because
+   one of the first messages right after boot is the device information)
+3. show a location trail of the device based purely on LTE network information
+4. show single cell (SCELL) vs. multi cell (MCELL) performance using nRF Cloud
+   Location services (these services can be purchased individually and have
+   different pricing: https://nrfcloud.com/#/pricing)

cdk/BackendLambdas.d.ts

@@ -14,4 +14,5 @@ type BackendLambdas = {
 	historicalDataRequest: PackedLambda
 	kpis: PackedLambda
 	configureDevice: PackedLambda
+	resolveSingleCellGeoLocation: PackedLambda
 }

cdk/packBackendLambdas.ts

@@ -39,4 +39,8 @@ export const packBackendLambdas = async (): Promise<BackendLambdas> => ({
 		'configureDevice',
 		'lambda/configureDevice.ts',
 	),
+	resolveSingleCellGeoLocation: await packLambdaFromPath(
+		'resolveSingleCellGeoLocation',
+		'lambda/resolveSingleCellGeoLocation.ts',
+	),
 })

cdk/resources/ConvertDeviceMessages.ts

@@ -4,13 +4,13 @@ import {
 	aws_iot as IoT,
 	aws_lambda as Lambda,
 	aws_logs as Logs,
-	Stack,
 } from 'aws-cdk-lib'
 import { Construct } from 'constructs'
 import type { PackedLambda } from '../helpers/lambdas/packLambda.js'
 import type { DeviceStorage } from './DeviceStorage.js'
 import { LambdaSource } from './LambdaSource.js'
 import type { WebsocketAPI } from './WebsocketAPI.js'
+import { IoTActionRole } from './IoTActionRole.js'
 
 /**
  * Resources needed to convert messages sent by nRF Cloud to the format that hello.nrfcloud.com expects
@@ -57,26 +57,6 @@ export class ConvertDeviceMessages extends Construct {
 		websocketAPI.eventBus.grantPutEventsTo(onDeviceMessage)
 		deviceStorage.devicesTable.grantReadData(onDeviceMessage)
 
-		const iotActionRole = new IAM.Role(this, 'iot-action-role', {
-			assumedBy: new IAM.ServicePrincipal(
-				'iot.amazonaws.com',
-			) as IAM.IPrincipal,
-			inlinePolicies: {
-				rootPermissions: new IAM.PolicyDocument({
-					statements: [
-						new IAM.PolicyStatement({
-							actions: ['iot:Publish'],
-							resources: [
-								`arn:aws:iot:${Stack.of(this).region}:${
-									Stack.of(this).account
-								}:topic/errors`,
-							],
-						}),
-					],
-				}),
-			},
-		})
-
 		const rule = new IoT.CfnTopicRule(this, 'topicRule', {
 			topicRulePayload: {
 				description: `Convert received message and publish to the EventBus`,
@@ -99,7 +79,7 @@ export class ConvertDeviceMessages extends Construct {
 				],
 				errorAction: {
 					republish: {
-						roleArn: iotActionRole.roleArn,
+						roleArn: new IoTActionRole(this).roleArn,
 						topic: 'errors',
 					},
 				},

cdk/resources/DeviceLastSeen.ts

@@ -1,11 +1,10 @@
 import {
 	aws_dynamodb as DynamoDB,
-	aws_iam as IAM,
 	aws_iot as IoT,
 	RemovalPolicy,
-	Stack,
 } from 'aws-cdk-lib'
 import { Construct } from 'constructs'
+import { IoTActionRole } from './IoTActionRole.js'
 
 /**
  * Record the timestamp when the device was last seen
@@ -46,25 +45,7 @@ export class DeviceLastSeen extends Construct {
 			projectionType: DynamoDB.ProjectionType.KEYS_ONLY,
 		})
 
-		const role = new IAM.Role(this, 'role', {
-			assumedBy: new IAM.ServicePrincipal(
-				'iot.amazonaws.com',
-			) as IAM.IPrincipal,
-			inlinePolicies: {
-				rootPermissions: new IAM.PolicyDocument({
-					statements: [
-						new IAM.PolicyStatement({
-							actions: ['iot:Publish'],
-							resources: [
-								`arn:aws:iot:${Stack.of(this).region}:${
-									Stack.of(this).account
-								}:topic/errors`,
-							],
-						}),
-					],
-				}),
-			},
-		})
+		const role = new IoTActionRole(this).role
 		this.table.grantWriteData(role)
 
 		new IoT.CfnTopicRule(this, 'rule', {

cdk/resources/IoTActionRole.ts

@@ -0,0 +1,33 @@
+import { aws_iam as IAM, Stack } from 'aws-cdk-lib'
+import { Construct } from 'constructs'
+
+/**
+ * Base role for IoT Actions that allows to publish to the 'errors' topic
+ */
+export class IoTActionRole extends Construct {
+	public readonly role: IAM.IRole
+	public readonly roleArn: string
+	constructor(parent: Construct) {
+		super(parent, 'errorActionRole')
+		this.role = new IAM.Role(this, 'iot-action-role', {
+			assumedBy: new IAM.ServicePrincipal(
+				'iot.amazonaws.com',
+			) as IAM.IPrincipal,
+			inlinePolicies: {
+				rootPermissions: new IAM.PolicyDocument({
+					statements: [
+						new IAM.PolicyStatement({
+							actions: ['iot:Publish'],
+							resources: [
+								`arn:aws:iot:${Stack.of(this).region}:${
+									Stack.of(this).account
+								}:topic/errors`,
+							],
+						}),
+					],
+				}),
+			},
+		})
+		this.roleArn = this.role.roleArn
+	}
+}

cdk/resources/SingleCellGeoLocation.ts

@@ -0,0 +1,115 @@
+import {
+	Duration,
+	aws_iam as IAM,
+	aws_iot as IoT,
+	aws_lambda as Lambda,
+	aws_logs as Logs,
+	Stack,
+} from 'aws-cdk-lib'
+import { Construct } from 'constructs'
+import type { PackedLambda } from '../helpers/lambdas/packLambda.js'
+import { LambdaSource } from './LambdaSource.js'
+import type { WebsocketAPI } from './WebsocketAPI.js'
+import { IoTActionRole } from './IoTActionRole.js'
+import type { DeviceStorage } from './DeviceStorage.js'
+
+/**
+ * Resolve device geo location based on network information
+ */
+export class SingleCellGeoLocation extends Construct {
+	public constructor(
+		parent: Construct,
+		{
+			lambdaSources,
+			layers,
+			websocketAPI,
+			deviceStorage,
+		}: {
+			websocketAPI: WebsocketAPI
+			deviceStorage: DeviceStorage
+			lambdaSources: {
+				resolveSingleCellGeoLocation: PackedLambda
+			}
+			layers: Lambda.ILayerVersion[]
+		},
+	) {
+		super(parent, 'SingleCellGeoLocation')
+
+		const fn = new Lambda.Function(this, 'fn', {
+			handler: lambdaSources.resolveSingleCellGeoLocation.handler,
+			architecture: Lambda.Architecture.ARM_64,
+			runtime: Lambda.Runtime.NODEJS_18_X,
+			timeout: Duration.seconds(60),
+			memorySize: 1792,
+			code: new LambdaSource(this, lambdaSources.resolveSingleCellGeoLocation)
+				.code,
+			description: 'Resolve device geo location based on network information',
+			environment: {
+				VERSION: this.node.tryGetContext('version'),
+				LOG_LEVEL: this.node.tryGetContext('logLevel'),
+				EVENTBUS_NAME: websocketAPI.eventBus.eventBusName,
+				NODE_NO_WARNINGS: '1',
+				DISABLE_METRICS: this.node.tryGetContext('isTest') === true ? '1' : '0',
+				STACK_NAME: Stack.of(this).stackName,
+				DEVICES_TABLE_NAME: deviceStorage.devicesTable.tableName,
+			},
+			layers,
+			logRetention: Logs.RetentionDays.ONE_WEEK,
+			initialPolicy: [
+				new IAM.PolicyStatement({
+					actions: ['ssm:GetParameter'],
+					resources: [
+						`arn:aws:ssm:${Stack.of(this).region}:${
+							Stack.of(this).account
+						}:parameter/${Stack.of(this).stackName}/thirdParty/nrfcloud/*`,
+					],
+				}),
+				new IAM.PolicyStatement({
+					actions: ['ssm:GetParametersByPath'],
+					resources: [
+						`arn:aws:ssm:${Stack.of(this).region}:${
+							Stack.of(this).account
+						}:parameter/${Stack.of(this).stackName}/thirdParty/nrfcloud`,
+					],
+				}),
+			],
+		})
+		websocketAPI.eventBus.grantPutEventsTo(fn)
+		deviceStorage.devicesTable.grantReadData(fn)
+
+		const rule = new IoT.CfnTopicRule(this, 'topicRule', {
+			topicRulePayload: {
+				description: `Resolve device geo location based on network information`,
+				ruleDisabled: false,
+				awsIotSqlVersion: '2016-03-23',
+				sql: `
+					select
+						* as message,
+						topic(4) as deviceId,
+						timestamp() as timestamp
+					from 'data/+/+/+/+'
+					where messageType = 'DATA'
+					and appId = 'DEVICE'
+				`,
+				actions: [
+					{
+						lambda: {
+							functionArn: fn.functionArn,
+						},
+					},
+				],
+				errorAction: {
+					republish: {
+						roleArn: new IoTActionRole(this).roleArn,
+						topic: 'errors',
+					},
+				},
+			},
+		})
+
+		fn.addPermission('topicRule', {
+			principal: new IAM.ServicePrincipal('iot.amazonaws.com'),
+			sourceArn: rule.attrArn,
+		})
+	}
+}

cdk/stacks/BackendStack.ts

@@ -26,6 +26,7 @@ import { WebsocketAPI } from '../resources/WebsocketAPI.js'
 import { KPIs } from '../resources/kpis/KPIs.js'
 import { STACK_NAME } from './stackConfig.js'
 import { ConfigureDevice } from '../resources/ConfigureDevice.js'
+import { SingleCellGeoLocation } from '../resources/SingleCellGeoLocation.js'
 
 export class BackendStack extends Stack {
 	public constructor(
@@ -165,6 +166,13 @@ export class BackendStack extends Stack {
 			websocketAPI,
 		})
 
+		new SingleCellGeoLocation(this, {
+			lambdaSources,
+			layers: lambdaLayers,
+			websocketAPI,
+			deviceStorage,
+		})
+
 		// Outputs
 		new CfnOutput(this, 'webSocketURI', {
 			exportName: `${this.stackName}:webSocketURI`,