diff --git a/packages/core/src/core.ts b/packages/core/src/core.ts index 70aaf12b4..a54e2e1e4 100644 --- a/packages/core/src/core.ts +++ b/packages/core/src/core.ts @@ -1285,7 +1285,7 @@ export class Core extends events.EventEmitter return value; } - getMachineId() { + static getMachineId() { try { return process.env["WEBDA_MACHINE_ID"] || machineIdSync(); /* c8 ignore next 4 */ @@ -1305,7 +1305,7 @@ export class Core extends events.EventEmitter type: "Webda/MemoryStore", persistence: { path: ".registry", - key: this.getMachineId() + key: Core.getMachineId() } }; this.createService(this.configuration.services, "Registry"); @@ -1635,8 +1635,8 @@ export class Core extends events.EventEmitter export type MetricConfiguration = T extends Counter ? CounterConfiguration : T extends Gauge - ? GaugeConfiguration - : HistogramConfiguration; + ? GaugeConfiguration + : HistogramConfiguration; /** * Export a Registry type alias diff --git a/packages/core/src/services/configuration.ts b/packages/core/src/services/configuration.ts index cb7b4f462..8e509aac6 100644 --- a/packages/core/src/services/configuration.ts +++ b/packages/core/src/services/configuration.ts @@ -43,8 +43,9 @@ export class ConfigurationServiceParameters extends ServiceParameters { } export type ConfigurationEvents = { - "Configuration.Applied": undefined; - "Configuration.Applying": undefined; + "Configuration.Applied": any; + "Configuration.Applying": any; + "Configuration.Loaded": any; }; /** @@ -204,9 +205,10 @@ export default class ConfigurationService< this.log("DEBUG", "Refreshing configuration"); const newConfig = (await this.loadConfiguration()) || this.parameters.default; + this.emit("Configuration.Loaded", newConfig); const serializedConfig = JSON.stringify(newConfig); if (serializedConfig !== this.serializedConfiguration) { - this.emit("Configuration.Applying", undefined); + this.emit("Configuration.Applying", newConfig); this.log("DEBUG", "Apply new configuration"); this.serializedConfiguration = serializedConfig; this.configuration = newConfig; @@ -228,7 +230,7 @@ export default class ConfigurationService< } }); await Promise.all(promises); - this.emit("Configuration.Applied", undefined); + this.emit("Configuration.Applied", newConfig); } // If the ConfigurationProvider cannot trigger we check at interval if (this.interval) { @@ -249,6 +251,7 @@ export default class ConfigurationService< */ async loadAndStoreConfiguration(): Promise<{ [key: string]: any }> { let res = await this.loadConfiguration(); + this.emit("Configuration.Loaded", res); this.serializedConfiguration = JSON.stringify(res); return res; } diff --git a/packages/core/src/services/cryptoservice.spec.ts b/packages/core/src/services/cryptoservice.spec.ts index 0ebfb78b3..912c468b2 100644 --- a/packages/core/src/services/cryptoservice.spec.ts +++ b/packages/core/src/services/cryptoservice.spec.ts @@ -4,7 +4,7 @@ import jwt from "jsonwebtoken"; import * as sinon from "sinon"; import { WebdaTest } from "../test"; import { JSONUtils } from "../utils/serializers"; -import CryptoService from "./cryptoservice"; +import CryptoService, { SecretString } from "./cryptoservice"; /** * @@ -124,3 +124,39 @@ class CryptoServiceTest extends WebdaTest { await assert.rejects(() => this.webda.getCrypto().getCurrentKeys(), /not initialized/); } } + +@suite +class CryptoConfigurationTest { + @test + async nominal() { + CryptoService.registerEncrypter("test", { + encrypt: async (data: any) => { + return data; + }, + decrypt: async (data: any) => { + return data; + } + }); + let data = { + key: { + secret: "encrypt:local:plop", + port: 21 + }, + anotherSecret: "encrypt:local:plop2", + notEncrypted: "plop", + alreadyEncrypted: "crypt:test:plop3" + }; + await CryptoService.encryptConfiguration(data); + assert.ok(data.key.secret.startsWith("crypt:local:")); + assert.ok(!data.key.secret.includes("plop")); + assert.ok(data.anotherSecret.startsWith("crypt:local:")); + assert.ok(!data.anotherSecret.includes("plop")); + assert.strictEqual(data.alreadyEncrypted, "crypt:test:plop3"); + let decrypted = await CryptoService.decryptConfiguration(JSONUtils.duplicate(data)); + assert.strictEqual(decrypted.anotherSecret.getValue(), "plop2"); + assert.strictEqual(decrypted.anotherSecret.toString(), "********"); + assert.strictEqual(SecretString.from(decrypted.alreadyEncrypted), "plop3"); + assert.strictEqual(SecretString.from(decrypted.key.secret), "plop"); + assert.strictEqual(`Test: ${decrypted.alreadyEncrypted}`, "Test: ********"); + } +} diff --git a/packages/core/src/services/cryptoservice.ts b/packages/core/src/services/cryptoservice.ts index adefaea89..ef8a76455 100644 --- a/packages/core/src/services/cryptoservice.ts +++ b/packages/core/src/services/cryptoservice.ts @@ -1,9 +1,41 @@ -import { createCipheriv, createDecipheriv, createHmac, generateKeyPairSync, randomBytes } from "crypto"; +import { createCipheriv, createDecipheriv, createHash, createHmac, generateKeyPairSync, randomBytes } from "crypto"; import jwt from "jsonwebtoken"; import { pem2jwk } from "pem-jwk"; -import { OperationContext, RegistryEntry, Store } from "../index"; +import * as util from "util"; +import { Core, OperationContext, RegistryEntry, Store } from "../index"; import { JSONUtils } from "../utils/serializers"; import { DeepPartial, Inject, Route, Service, ServiceParameters } from "./service"; + +export class SecretString { + constructor( + protected str: string, + protected encrypter: string + ) {} + + static from(value: string | SecretString, path?: string): string { + if (value instanceof SecretString) { + return value.getValue(); + } + if (Core.get()) { + Core.get()?.log("WARN", "A secret string is not encrypted", value); + } else { + console.error("WARN", "A secret string is not encrypted"); + } + + return value; + } + + getValue(): string { + return this.str; + } + toString(): string { + return "********"; + } + [util.inspect.custom](depth, options, inspect) { + return "********"; + } +} + export interface KeysRegistry { /** * Contains the instanceId of the last @@ -24,6 +56,24 @@ export interface KeysRegistry { current: string; } +/** + * Encrypt/Decrypt string + */ +export interface StringEncrypter { + /** + * Encrypt a string + * @param data + * @returns + */ + encrypt(data: string, options?: any): Promise; + /** + * Decrypt a string + * @param data + * @returns + */ + decrypt(data: string, options?: any): Promise; +} + /** * JWT Options */ @@ -176,7 +226,26 @@ interface KeysDefinition { /** * @WebdaModda */ -export default class CryptoService extends Service { +export default class CryptoService + extends Service + implements StringEncrypter +{ + private static encrypters: { [key: string]: StringEncrypter } = {}; + + /** + * Register an encrypter for configuration + * @param name + * @param encrypter + */ + static registerEncrypter( + name: string, + encrypter: { encrypt: (data: string) => Promise; decrypt: (data: string) => Promise } + ) { + if (CryptoService.encrypters[name]) { + console.error("Encrypter", name, "already registered"); + } + CryptoService.encrypters[name] = encrypter; + } currentSymetricKey: string; currentAsymetricKey: { publicKey: string; privateKey: string }; current: string; @@ -209,6 +278,7 @@ export default class CryptoService { await super.init(); + CryptoService.encrypters["self"] = this; // Load keys if (!(await this.load()) && this.parameters.autoCreate) { await this.rotate(); @@ -437,19 +507,74 @@ export default class CryptoService { + if (data instanceof Object) { + for (let i in data) { + data[i] = await CryptoService.decryptConfiguration(data[i]); + } + } else if (typeof data === "string") { + if (data.startsWith("crypt:") || data.startsWith("scrypt:")) { + let str = data.substring(data.indexOf(":") + 1); + let type = str.substring(0, str.indexOf(":")); + str = str.substring(str.indexOf(":") + 1); + if (!CryptoService.encrypters[type]) { + throw new Error("Unknown encrypter " + type); + } + // We keep the ability to map to a simple string for incompatible module + if (data.startsWith("scrypt:")) { + return await CryptoService.encrypters[type].decrypt(str); + } else { + return new SecretString(await CryptoService.encrypters[type].decrypt(str), type); + } + } + } + return data; + } + /** * Decrypt data */ public async decrypt(token: string): Promise { let input = Buffer.from(await this.jwtVerify(token), "base64"); let header = this.getJWTHeader(token); - let iv = input.slice(0, 16); + let iv = input.subarray(0, 16); let decipher = createDecipheriv( this.parameters.symetricCipher, Buffer.from(this.keys[header.kid.substring(1)].symetric, "base64"), iv ); - return JSON.parse(decipher.update(input.slice(16)).toString() + decipher.final().toString()); + return JSON.parse(decipher.update(input.subarray(16)).toString() + decipher.final().toString()); } /** @@ -490,4 +615,24 @@ export default class CryptoService { + // Initialization Vector + let iv = randomBytes(16); + const key = createHash("sha256").update(Core.getMachineId()).digest(); + let cipher = createCipheriv("aes-256-ctr", key, iv); + return Buffer.concat([iv, cipher.update(Buffer.from(data)), cipher.final()]).toString("base64"); + }, + decrypt: async (data: string) => { + let input = Buffer.from(data, "base64"); + let iv = input.subarray(0, 16); + const key = createHash("sha256").update(Core.getMachineId()).digest(); + let decipher = createDecipheriv("aes-256-ctr", key, iv); + return decipher.update(input.subarray(16)).toString() + decipher.final().toString(); + } +}); + export { CryptoService }; diff --git a/packages/core/src/utils/serializers.spec.ts b/packages/core/src/utils/serializers.spec.ts index 51edfc26e..494d0e012 100644 --- a/packages/core/src/utils/serializers.spec.ts +++ b/packages/core/src/utils/serializers.spec.ts @@ -1,6 +1,14 @@ import { suite, test } from "@testdeck/mocha"; import * as assert from "assert"; -import { createReadStream, createWriteStream, existsSync, readFileSync, symlinkSync, unlinkSync } from "fs"; +import { + createReadStream, + createWriteStream, + existsSync, + readFileSync, + symlinkSync, + unlinkSync, + writeFileSync +} from "fs"; import * as path from "path"; import { pipeline } from "stream/promises"; import { fileURLToPath } from "url"; @@ -278,4 +286,58 @@ plop: test await p; FileUtils.getReadStream("/tmp/webda.stream"); } + + @test + jsoncUpdateFile() { + const file = "/tmp/webda.jsonc"; + const JSONC_SOURCE = `{ + "test": "plop", + // Comment one + "test3": { + "bouzoufr": "plop" + }, + "test4": { + "array": [ + "plop", + "plop2", + {"id": "plop3"} + ], + "p": { + "v": "bouzouf" + }, /* c, */ + "remove": true /* c */ + }, + "test2": { + "plop3": "bouzouf", // Comment on 3 + /* */ /* */ /* */ /* */ /* */ /* */ + /* + Test of , comments + */ + "plop2": "bouzouf", // Comment two with a , for fun + /** + * another , one + * */ + "plop4": "bouzouf" + }, + "removed": true + }`; + try { + writeFileSync(file, JSONC_SOURCE); + JSONUtils.updateFile(file, v => { + if (v === "bouzouf") { + return "bouzouf2"; + } + return v; + }); + const data = FileUtils.load(file); + assert.strictEqual(data.test4.p.v, "bouzouf2"); + assert.strictEqual(data.test2.plop4, "bouzouf2"); + assert.strictEqual(data.test2.plop2, "bouzouf2"); + assert.strictEqual(data.test2.plop3, "bouzouf2"); + const update = readFileSync(file).toString(); + assert.strictEqual(update.length, JSONC_SOURCE.length + 4); + } finally { + FileUtils.clean(file); + } + } } diff --git a/packages/core/src/utils/serializers.ts b/packages/core/src/utils/serializers.ts index 1e3de59bd..6e818b28e 100644 --- a/packages/core/src/utils/serializers.ts +++ b/packages/core/src/utils/serializers.ts @@ -398,6 +398,30 @@ export const JSONUtils = { duplicate: value => { return JSON.parse(JSONUtils.stringify(value)); }, + /** + * Visit a json/jsonc file for update + * @param filename + */ + updateFile: async (filename: string, replacer: (value: any) => any) => { + const content = readFileSync(filename).toString().trim(); + const edits: jsonc.EditResult = []; + const promises: Promise[] = []; + jsonc.visit(content, { + onLiteralValue(value, offset, length, startLine, startCharacter, pathSupplier) { + promises.push( + (async () => { + const newValue = await replacer(value); + if (value !== newValue) { + edits.push({ offset, length, content: JSON.stringify(newValue, undefined, 2) }); + } + })() + ); + } + }); + // + await Promise.all(promises); + writeFileSync(filename, jsonc.applyEdits(content, edits)); + }, /** * Helper to FileUtils.save */ diff --git a/packages/gcp/src/services/kms.spec.ts b/packages/gcp/src/services/kms.spec.ts new file mode 100644 index 000000000..70f7ab425 --- /dev/null +++ b/packages/gcp/src/services/kms.spec.ts @@ -0,0 +1,45 @@ +import { KeyManagementServiceClient } from "@google-cloud/kms"; +import { suite, test } from "@testdeck/mocha"; +import { WebdaSimpleTest } from "@webda/core/lib/test"; +import * as assert from "assert"; +import * as sinon from "sinon"; +import { GCPKMSService } from "./kms"; + +@suite +class KMSTest extends WebdaSimpleTest { + @test + async test() { + sinon.stub(KeyManagementServiceClient.prototype, "encrypt").callsFake(() => { + return [ + { + ciphertext: "ciphertext" + } + ]; + }); + sinon.stub(KeyManagementServiceClient.prototype, "decrypt").callsFake(() => { + return [ + { + plaintext: "plaintext" + } + ]; + }); + let service = await this.registerService( + new GCPKMSService(this.webda, "KMSService", { + defaultKey: "projects/my-project/locations/us-east1/keyRings/my-key-ring/cryptoKeys/my-key" + }) + ) + .resolve() + .init(); + let encoded = await service.encrypt("test"); + assert.deepStrictEqual(Buffer.from(encoded.split(":")[0], "base64").toString().split(":"), [ + "my-project", + "us-east1", + "my-key-ring", + "my-key" + ]); + assert.strictEqual(encoded.split(":").pop(), "ciphertext"); + let decoded = await service.decrypt(encoded); + assert.strictEqual(decoded, "plaintext"); + assert.rejects(() => service.decrypt(Buffer.from("test:plop").toString("base64") + ":test")); + } +} diff --git a/packages/gcp/src/services/kms.ts b/packages/gcp/src/services/kms.ts new file mode 100644 index 000000000..1d2d1ed72 --- /dev/null +++ b/packages/gcp/src/services/kms.ts @@ -0,0 +1,96 @@ +import { KeyManagementServiceClient } from "@google-cloud/kms"; +import { CryptoService, DeepPartial, Service, ServiceParameters } from "@webda/core"; + +/** + * Encrypter for GCP KMS + */ +const encrypter = { + encrypt: async (data: string, key?: string): Promise => { + key ??= process.env.WEBDA_GCP_KMS_KEY; + // Only the odd elements are dynamic + const infos = key.split("/").filter((i, ind) => ind % 2 === 1); + const client = new KeyManagementServiceClient(); + // Encode the infos in base64 + // Add the encrypted info after a : separator + return ( + Buffer.from(infos.join(":")).toString("base64") + + ":" + + ( + await client.encrypt({ + name: key, + plaintext: Buffer.from(data) + }) + )[0].ciphertext + ); + }, + decrypt: async (data: string): Promise => { + // Get the info + const info = data.substring(0, data.indexOf(":")); + const infos = Buffer.from(info, "base64").toString().split(":"); + if (infos.length !== 4) { + throw new Error("Invalid KMS encryption"); + } + const client = new KeyManagementServiceClient(); + return ( + await client.decrypt({ + name: `projects/${infos[0]}/locations/${infos[1]}/keyRings/${infos[2]}/cryptoKeys/${infos[3]}`, + ciphertext: Buffer.from(data.substring(data.indexOf(":") + 1)) + }) + )[0].plaintext; + } +}; + +/** + * Register the encrypter + */ +CryptoService.registerEncrypter("gcp", encrypter); + +/** + * Parameters for the KMS Service + */ +export class KMSServiceParameters extends ServiceParameters { + /** + * Encryption key to use by default + * @default WEBDA_GCP_KMS_KEY env variable + */ + defaultKey?: string; + constructor(params: any) { + super(params); + this.defaultKey ??= process.env.WEBDA_GCP_KMS_KEY; + } +} + +/** + * Expose KMS Service + * + * @WebdaModda GoogleCloudKMS + */ +export class GCPKMSService extends Service { + client: KeyManagementServiceClient; + + /** + * @override + */ + loadParameters(params: DeepPartial): ServiceParameters { + return new KMSServiceParameters(params); + } + + /** + * Encrypt a data with GCP KMS given key or defaultKey + * @param data + * @param key + * @returns + */ + encrypt(data: string, key?: string): Promise { + return encrypter.encrypt(data, key || this.parameters.defaultKey); + } + + /** + * Decrypt a data previously encrypted with this service + * @param data + * @returns + */ + decrypt(data: string): Promise { + return encrypter.decrypt(data); + } +} diff --git a/packages/gcp/webda.module.json b/packages/gcp/webda.module.json index bfc524346..0d7f09dd2 100644 --- a/packages/gcp/webda.module.json +++ b/packages/gcp/webda.module.json @@ -3,6 +3,7 @@ "deployers": {}, "moddas": { "Webda/GoogleCloudFireStore": "lib/services/firestore:default", + "Webda/GoogleCloudKMS": "lib/services/kms:GCPKMSService", "Webda/GoogleCloudPubSub": "lib/services/pubsub:default", "Webda/GoogleCloudQueue": "lib/services/queue:default", "Webda/GoogleCloudStorage": "lib/services/storage:default" @@ -115,6 +116,10 @@ }, "description": "Model Aliases to allow easier rename of Model" }, + "noCache": { + "type": "boolean", + "description": "Disable default memory cache" + }, "collection": { "type": "string", "description": "Collection to use" @@ -202,6 +207,34 @@ }, "title": "FireStore" }, + "Webda/GoogleCloudKMS": { + "type": "object", + "properties": { + "type": { + "type": "string", + "description": "Type of the service" + }, + "url": { + "type": "string", + "description": "URL on which to serve the content" + }, + "defaultKey": { + "type": "string", + "description": "Encryption key to use by default", + "default": "WEBDA_GCP_KMS_KEY env variable" + }, + "openapi": { + "type": "object", + "additionalProperties": true + } + }, + "required": [ + "type" + ], + "description": "Parameters for the KMS Service", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "GCPKMSService" + }, "Webda/GoogleCloudPubSub": { "type": "object", "properties": { diff --git a/packages/runtime/package.json b/packages/runtime/package.json index 717422fad..012126a70 100644 --- a/packages/runtime/package.json +++ b/packages/runtime/package.json @@ -41,6 +41,7 @@ "node-machine-id": "^1.1.12", "nodemailer": "^6.6.3", "openapi-types": "^12.0.0", + "password-prompt": "^1.1.3", "pem-jwk": "^2.0.0", "prom-client": "^15.0.0", "sanitize-html": "^2.4.0", @@ -52,9 +53,9 @@ "yaml": "^2.0.0" }, "devDependencies": { - "@webda/shell": "^3.8.2", "@testdeck/mocha": "^0.3.2", "@types/node": "18.11.13", + "@webda/shell": "^3.8.2", "c8": "^8.0.0", "fs-extra": "^11.0.0", "mocha": "^10.0.0", @@ -96,4 +97,4 @@ "engines": { "node": ">=18.0.0" } -} \ No newline at end of file +} diff --git a/packages/runtime/src/utils/password.ts b/packages/runtime/src/utils/password.ts new file mode 100644 index 000000000..2f9c7eaae --- /dev/null +++ b/packages/runtime/src/utils/password.ts @@ -0,0 +1,87 @@ +import { Core, CryptoService, DeepPartial, Service, ServiceParameters, StringEncrypter } from "@webda/core"; +import { WorkerInputType } from "@webda/workout"; +import { createCipheriv, createDecipheriv, randomBytes, scryptSync } from "node:crypto"; +/** + * Inputted password + */ +let encryptPassword; + +/** + * Derive a key from a password + * @param password + * @returns + */ +function getKey(password: string): Buffer { + return scryptSync(password, "Webda", 64).subarray(0, 32); +} + +async function requestPassword(): Promise { + if (!Core.get().getWorkerOutput().interactive && process.stdin.isTTY) { + // Fallback to password-prompt as we have a tty + let passwordLib = await import("password-prompt"); + return passwordLib.default("Configuration Encryption Password: ", { method: "hide" }); + } + return await Core.get() + .getWorkerOutput() + .requestInput("Configuration Encryption Password", WorkerInputType.PASSWORD, [], true); +} + +const encrypter: StringEncrypter = { + encrypt: async (data: string, password?: string): Promise => { + if (!password) { + encryptPassword ??= await requestPassword(); + password = encryptPassword; + } + // Derive key TODO replace by a true derivation function + let iv = randomBytes(16); + let cipher = createCipheriv("aes-256-ctr", getKey(password), iv); + return Buffer.concat([iv, cipher.update(Buffer.from(data)), cipher.final()]).toString("base64"); + }, + decrypt: async (data: string, password?: string): Promise => { + if (!password) { + encryptPassword ??= await requestPassword(); + password = encryptPassword; + } + let input = Buffer.from(data, "base64"); + let iv = input.subarray(0, 16); + let decipher = createDecipheriv("aes-256-ctr", getKey(password), iv); + return decipher.update(input.subarray(16)).toString() + decipher.final().toString(); + } +}; +/** + * Register a password encrypted data + */ +CryptoService.registerEncrypter("password", encrypter); + +/** + * Webda Service to encrypt password + * + * @WebdaModda + */ +export class PasswordEncryptionService extends Service { + /** + * @override + */ + loadParameters(params: DeepPartial): ServiceParameters { + return new ServiceParameters(params); + } + + /** + * Encrypt a data with GCP KMS given key or defaultKey + * @param data + * @param key + * @returns + */ + encrypt(data: string, key?: string): Promise { + return encrypter.encrypt(data, key); + } + + /** + * Decrypt a data previously encrypted with this service + * @param data + * @returns + */ + decrypt(data: string, key?: string): Promise { + return encrypter.decrypt(data, key); + } +} diff --git a/packages/shell/src/console/webda.ts b/packages/shell/src/console/webda.ts index cf0e3a4b8..fa6f4231b 100644 --- a/packages/shell/src/console/webda.ts +++ b/packages/shell/src/console/webda.ts @@ -1,5 +1,6 @@ import { CancelablePromise, + CryptoService, FileUtils, getCommonJS, JSONUtils, @@ -576,6 +577,38 @@ ${Object.keys(operationsExport.operations) return 0; } + /** + * If deployment in argument: display or export the configuration + * Otherwise launch the configuration UI + * + * @param argv + */ + static async configEncrypt(argv: yargs.Arguments): Promise { + const filename = argv.file; + if (!filename.match(/\.jsonc?$/)) { + this.log("ERROR", "Only json/jsonc format are handled for now"); + return -1; + } + this.log("INFO", "Encrypting values in configuration file", filename); + WebdaConsole.webda = new WebdaServer(this.app); + await WebdaConsole.webda.init(); + await JSONUtils.updateFile(filename, async value => { + if (typeof value === "string") { + // We want to migrate all encrypted string to another type of encryption + if (argv.migrate) { + let newValue = await CryptoService.decryptConfiguration(value); + if (value !== newValue) { + value = `encrypt:${argv.migrate}:${newValue.getValue()}`; + } + } + return await CryptoService.encryptConfiguration(value); + } else { + return value; + } + }); + return 0; + } + /** * Rotate crypto keys */ @@ -832,7 +865,7 @@ ${Object.keys(operationsExport.operations) }, websockets: { alias: "w", - deprecated: "websockets can be enable by adding specific modules now", + deprecated: "websockets can be enable by adding specific modules now" } } }, @@ -910,6 +943,16 @@ ${Object.keys(operationsExport.operations) return y.command("exportFile", "File to export configuration to"); } }, + "config-encrypt": { + handler: WebdaConsole.configEncrypt, + command: "config-encrypt [file]", + description: "Encrypt all fields due for encryption in the file", + module: { + migrate: { + type: "string" + } + } + }, init: { command: "init [generator]", handler: WebdaConsole.init,