diff --git a/.github/workflows/test-and-release.yaml b/.github/workflows/test-and-release.yaml index 1c0379403..e8ac47f39 100644 --- a/.github/workflows/test-and-release.yaml +++ b/.github/workflows/test-and-release.yaml @@ -107,6 +107,7 @@ jobs: ./cli.sh fake-nrfcloud-account-device --remove ./cli.sh configure thirdParty/nrfcloud/apiEndpoint -X ./cli.sh configure thirdParty/nrfcloud/apiKey -X + ./cli.sh clean-backup-certificates release: needs: diff --git a/cdk/backend.ts b/cdk/backend.ts index f5b456d4b..34a565d4d 100644 --- a/cdk/backend.ts +++ b/cdk/backend.ts @@ -2,20 +2,25 @@ import { ECRClient } from '@aws-sdk/client-ecr' import { IAMClient } from '@aws-sdk/client-iam' import { IoTClient } from '@aws-sdk/client-iot' import { STS } from '@aws-sdk/client-sts' +import { SSMClient } from '@aws-sdk/client-ssm' import path from 'node:path' import { getIoTEndpoint } from '../aws/getIoTEndpoint.js' import { getOrBuildDockerImage } from '../aws/getOrBuildDockerImage.js' import { getOrCreateRepository } from '../aws/getOrCreateRepository.js' import { ensureCA } from '../bridge/ensureCA.js' import { ensureMQTTBridgeCredentials } from '../bridge/ensureMQTTBridgeCredentials.js' -import { debug } from '../cli/log.js' +import { debug, type logFn } from '../cli/log.js' import pJSON from '../package.json' import { BackendApp } from './BackendApp.js' import { ensureGitHubOIDCProvider } from './ensureGitHubOIDCProvider.js' import { env } from './helpers/env.js' import { packLayer } from './helpers/lambdas/packLayer.js' import { packBackendLambdas } from './packBackendLambdas.js' -import { ECR_NAME } from './stacks/stackConfig.js' +import { ECR_NAME, STACK_NAME } from './stacks/stackConfig.js' +import { mkdir } from 'node:fs/promises' +import { Scope, getSettingsOptional, putSettings } from '../util/settings.js' +import { readFilesFromMap } from './helpers/readFilesFromMap.js' +import { writeFilesFromMap } from './helpers/writeFilesFromMap.js' const repoUrl = new URL(pJSON.repository.url) const repository = { @@ -27,6 +32,7 @@ const iot = new IoTClient({}) const sts = new STS({}) const ecr = new ECRClient({}) const iam = new IAMClient({}) +const ssm = new SSMClient({}) const accountEnv = await env({ sts }) @@ -49,16 +55,86 @@ const certsDir = path.join( 'certificates', `${accountEnv.account}@${accountEnv.region}`, ) +await mkdir(certsDir, { recursive: true }) +const mqttBridgeDebug = debug('MQTT bridge') +const caDebug = debug('CA certificate') const mqttBridgeCertificate = await ensureMQTTBridgeCredentials({ iot, certsDir, - debug: debug('MQTT bridge'), + debug: mqttBridgeDebug, })() const caCertificate = await ensureCA({ certsDir, iot, - debug: debug('CA certificate'), + debug: caDebug, })() +const restoreCertsFromSettings = + (certsMap: Record, debug: logFn) => + async (settings: null | Record): Promise => { + if (settings === null) return false + const locations: Record = Object.entries(settings).reduce( + (locations, [k, v]) => { + const path = certsMap[k] + debug(`Unrecognized path:`, k) + if (path === undefined) return locations + return { + ...locations, + [path]: v, + } + }, + {}, + ) + // Make sure all required locations exist + for (const k of Object.keys(certsMap)) { + if (locations[k] === undefined) { + debug(`Restored certificate settings are missing key`, k) + return false + } + } + for (const k of Object.keys(locations)) debug(`Restoring:`, k) + await writeFilesFromMap(settings) + return true + } +const useRestoredCertificates = await Promise.all([ + getSettingsOptional, null>({ + ssm, + stackName: STACK_NAME, + scope: Scope.NRFCLOUD_BRIDGE_CERTIFICATE_MQTT, + })(null).then( + restoreCertsFromSettings(mqttBridgeCertificate, mqttBridgeDebug), + ), + getSettingsOptional, null>({ + ssm, + stackName: STACK_NAME, + scope: Scope.NRFCLOUD_BRIDGE_CERTIFICATE_CA, + })(null).then(restoreCertsFromSettings(caCertificate, caDebug)), +]) + +if (!useRestoredCertificates.some(Boolean)) { + await Promise.all([ + Object.entries(readFilesFromMap(mqttBridgeCertificate)).map( + async ([k, v]) => + putSettings({ + ssm, + stackName: STACK_NAME, + scope: Scope.NRFCLOUD_BRIDGE_CERTIFICATE_MQTT, + })({ + property: k, + value: v, + }), + ), + Object.entries(readFilesFromMap(caCertificate)).map(async ([k, v]) => + putSettings({ + ssm, + stackName: STACK_NAME, + scope: Scope.NRFCLOUD_BRIDGE_CERTIFICATE_CA, + })({ + property: k, + value: v, + }), + ), + ]) +} // Prebuild / reuse docker image // NOTE: It is intention that release image tag can be undefined during the development, diff --git a/cdk/helpers/readFilesFromMap.spec.ts b/cdk/helpers/readFilesFromMap.spec.ts new file mode 100644 index 000000000..df7511ae1 --- /dev/null +++ b/cdk/helpers/readFilesFromMap.spec.ts @@ -0,0 +1,26 @@ +import path from 'node:path' +import os from 'node:os' +import fs from 'node:fs/promises' +import { readFilesFromMap } from './readFilesFromMap.js' + +describe('readFilesFromMap()', () => { + it('should read files from a map', async () => { + const tempDir = await fs.mkdtemp( + path.join(os.tmpdir(), 'readfilesfrommap-'), + ) + const f1 = path.join(tempDir, 'f1.txt') + const f2 = path.join(tempDir, 'f2.txt') + await fs.writeFile(f1, 'f1', 'utf-8') + await fs.writeFile(f2, 'f2', 'utf-8') + + expect( + await readFilesFromMap({ + f1, + f2, + }), + ).toMatchObject({ + f1: 'f1', + f2: 'f2', + }) + }) +}) diff --git a/cdk/helpers/readFilesFromMap.ts b/cdk/helpers/readFilesFromMap.ts new file mode 100644 index 000000000..0b9ed4d48 --- /dev/null +++ b/cdk/helpers/readFilesFromMap.ts @@ -0,0 +1,16 @@ +import { readFile } from 'node:fs/promises' + +export const readFilesFromMap = async ( + fileMap: Record, +): Promise> => { + const contents = await Promise.all( + Object.entries(fileMap).map>( + async ([key, path]) => [key, await readFile(path, 'utf-8')], + ), + ) + + return contents.reduce( + (contentsMap, [key, content]) => ({ ...contentsMap, [key]: content }), + {}, + ) +} diff --git a/cdk/helpers/writeFilesFromMap.spec.ts b/cdk/helpers/writeFilesFromMap.spec.ts new file mode 100644 index 000000000..95c9e6909 --- /dev/null +++ b/cdk/helpers/writeFilesFromMap.spec.ts @@ -0,0 +1,22 @@ +import path from 'node:path' +import os from 'node:os' +import fs from 'node:fs/promises' +import { writeFilesFromMap } from './writeFilesFromMap.js' + +describe('writeFilesFromMap()', () => { + it('should read files from a map', async () => { + const tempDir = await fs.mkdtemp( + path.join(os.tmpdir(), 'writeFilesFromMap-'), + ) + const f1 = path.join(tempDir, 'f1.txt') + const f2 = path.join(tempDir, 'f2.txt') + + await writeFilesFromMap({ + [f1]: 'f1', + [f2]: 'f2', + }) + + expect(await fs.readFile(f1, 'utf-8')).toEqual('f1') + expect(await fs.readFile(f2, 'utf-8')).toEqual('f2') + }) +}) diff --git a/cdk/helpers/writeFilesFromMap.ts b/cdk/helpers/writeFilesFromMap.ts new file mode 100644 index 000000000..97c98c137 --- /dev/null +++ b/cdk/helpers/writeFilesFromMap.ts @@ -0,0 +1,11 @@ +import { writeFile } from 'node:fs/promises' + +export const writeFilesFromMap = async ( + fileContents: Record, +): Promise => { + await Promise.all( + Object.entries(fileContents).map(async ([path, contents]) => + writeFile(path, contents, 'utf-8'), + ), + ) +} diff --git a/cdk/resources/HealthCheckMqttBridge.ts b/cdk/resources/HealthCheckMqttBridge.ts index 3389439c3..39f0ddac6 100644 --- a/cdk/resources/HealthCheckMqttBridge.ts +++ b/cdk/resources/HealthCheckMqttBridge.ts @@ -12,6 +12,7 @@ import { type Settings as BridgeSettings } from '../../bridge/settings.js' import type { PackedLambda } from '../helpers/lambdas/packLambda.js' import type { DeviceStorage } from './DeviceStorage.js' import type { WebsocketAPI } from './WebsocketAPI.js' +import { LambdaSource } from './LambdaSource.js' export type BridgeImageSettings = BridgeSettings @@ -46,7 +47,7 @@ export class HealthCheckMqttBridge extends Construct { runtime: Lambda.Runtime.NODEJS_18_X, timeout: Duration.seconds(15), memorySize: 1792, - code: Lambda.Code.fromAsset(lambdaSources.healthCheck.zipFile), + code: new LambdaSource(this, lambdaSources.healthCheck).code, description: 'End to end test for mqtt bridge', environment: { VERSION: this.node.tryGetContext('version'), diff --git a/cdk/stacks/BackendStack.ts b/cdk/stacks/BackendStack.ts index bbd14f184..75fb6cdc7 100644 --- a/cdk/stacks/BackendStack.ts +++ b/cdk/stacks/BackendStack.ts @@ -86,7 +86,11 @@ export class BackendStack extends Stack { this, 'healthCheckLayer', { - code: Lambda.Code.fromAsset(healthCheckLayer.layerZipFile), + code: new LambdaSource(this, { + id: 'healthcheckLayer', + zipFile: healthCheckLayer.layerZipFile, + hash: healthCheckLayer.hash, + }).code, compatibleArchitectures: [Lambda.Architecture.ARM_64], compatibleRuntimes: [Lambda.Runtime.NODEJS_18_X], }, diff --git a/cli/cli.ts b/cli/cli.ts index 038f96fab..8441972d0 100644 --- a/cli/cli.ts +++ b/cli/cli.ts @@ -28,6 +28,7 @@ import { showDeviceCommand } from './commands/show-device.js' import { showFingerprintCommand } from './commands/show-fingerprint.js' import { showNRFCloudAccount } from './commands/show-nrfcloud-account.js' import { simulateDeviceCommand } from './commands/simulate-device.js' +import { cleanBackupCertificates } from './commands/clean-backup-certificates.js' const ssm = new SSMClient({}) const iot = new IoTClient({}) @@ -61,6 +62,7 @@ const CLI = async ({ isCI }: { isCI: boolean }) => { configureCommand({ ssm }), setShadowFetcherCommand({ ssm }), logsCommand({ stackName: STACK_NAME, cf, logs }), + cleanBackupCertificates({ ssm }), ] if (isCI) { diff --git a/cli/commands/clean-backup-certificates.ts b/cli/commands/clean-backup-certificates.ts new file mode 100644 index 000000000..b0fd48056 --- /dev/null +++ b/cli/commands/clean-backup-certificates.ts @@ -0,0 +1,46 @@ +import { + DeleteParametersCommand, + GetParametersByPathCommand, + SSMClient, +} from '@aws-sdk/client-ssm' +import chalk from 'chalk' +import { chunk } from 'lodash-es' +import { STACK_NAME } from '../../cdk/stacks/stackConfig.js' +import { Scope, settingsPath } from '../../util/settings.js' +import type { CommandDefinition } from './CommandDefinition.js' + +export const cleanBackupCertificates = ({ + ssm, +}: { + ssm: SSMClient +}): CommandDefinition => ({ + command: 'clean-backup-certificates', + action: async () => { + console.debug(chalk.magenta(`Deleting backup certificates`)) + for (const scope of [ + Scope.NRFCLOUD_BRIDGE_CERTIFICATE_CA, + Scope.NRFCLOUD_BRIDGE_CERTIFICATE_MQTT, + ]) { + const parameters = await ssm.send( + new GetParametersByPathCommand({ + Path: settingsPath({ + stackName: STACK_NAME, + scope, + }), + Recursive: true, + }), + ) + + const names = [...(parameters.Parameters?.map((p) => p.Name) ?? [])] + const namesChunk = chunk(names, 10) + for (const names of namesChunk) { + await ssm.send( + new DeleteParametersCommand({ + Names: names as string[], + }), + ) + } + } + }, + help: 'Clean backup certificates on SSM', +}) diff --git a/util/settings.spec.ts b/util/settings.spec.ts index 4db0ee2db..c1e0580ae 100644 --- a/util/settings.spec.ts +++ b/util/settings.spec.ts @@ -1,5 +1,10 @@ import type { SSMClient } from '@aws-sdk/client-ssm' -import { Scope, getSettingsOptional, settingsPath } from './settings.js' +import { + Scope, + getSettingsOptional, + settingsPath, + getSettings, +} from './settings.js' describe('getSettingsOptional()', () => { it('should return the given default value if parameter does not exist', async () => { @@ -29,3 +34,37 @@ describe('settingsPath()', () => { }), ).toEqual('/hello-nrfcloud/stack/context/someProperty')) }) + +describe('getSettings()', () => { + it('should return the object with same scope', async () => { + const returnedValues = [ + { + Name: `/hello-nrfcloud/stack/context/key1`, + Value: 'value1', + }, + { + Name: `/hello-nrfcloud/stack/context/key2`, + Value: 'value2', + }, + { + Name: `/hello-nrfcloud/stack/context/key3`, + Value: 'value3', + }, + ] + + const stackConfig = getSettings({ + ssm: { + send: jest.fn().mockResolvedValue({ Parameters: returnedValues }), + } as unknown as SSMClient, + stackName: 'hello-nrfcloud', + scope: Scope.STACK_CONFIG, + }) + + const result = await stackConfig() + expect(result).toEqual({ + key1: 'value1', + key2: 'value2', + key3: 'value3', + }) + }) +}) diff --git a/util/settings.ts b/util/settings.ts index c1120f7a4..57f93a274 100644 --- a/util/settings.ts +++ b/util/settings.ts @@ -10,6 +10,8 @@ import { paginate } from './paginate.js' export enum Scope { STACK_CONFIG = 'stack/context', NRFCLOUD_CONFIG = 'thirdParty/nrfcloud', + NRFCLOUD_BRIDGE_CERTIFICATE_MQTT = 'nRFCloudBridgeCertificate/MQTT', + NRFCLOUD_BRIDGE_CERTIFICATE_CA = 'nRFCloudBridgeCertificate/CA', } export const settingsPath = ({