From 8f9d908dbe5878369638e8a74ffea3b96b039fc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20K=C3=B6nig?= Date: Thu, 17 Jul 2025 17:32:31 +0200 Subject: [PATCH 1/5] chore: remove unneeded device migrations --- .../sync/migrations/DeviceMigrations.ts | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/packages/transport/src/modules/sync/migrations/DeviceMigrations.ts b/packages/transport/src/modules/sync/migrations/DeviceMigrations.ts index b5c5278e6..9794a7086 100644 --- a/packages/transport/src/modules/sync/migrations/DeviceMigrations.ts +++ b/packages/transport/src/modules/sync/migrations/DeviceMigrations.ts @@ -1,26 +1,9 @@ -import { CoreSynchronizable } from "../../../core/CoreSynchronizable"; import { AccountController } from "../../accounts/AccountController"; export class DeviceMigrations { public constructor(private readonly accountController: AccountController) {} public async v1(): Promise { - const query = { cache: { $exists: false } }; - const synchronizableToId = (c: CoreSynchronizable) => c.id.toString(); - - const files = await this.accountController.files.getFiles(query); - await this.accountController.files.updateCache(files.map(synchronizableToId)); - - const messages = await this.accountController.messages.getMessages(query); - await this.accountController.messages.updateCache(messages.map(synchronizableToId)); - - const relationships = await this.accountController.relationships.getRelationships(query); - await this.accountController.relationships.updateCache(relationships.map(synchronizableToId)); - - const templates = await this.accountController.relationshipTemplates.getRelationshipTemplates(query); - await this.accountController.relationshipTemplates.updateCache(templates.map(synchronizableToId)); - - const tokens = await this.accountController.tokens.getTokens(query); - await this.accountController.tokens.updateCache(tokens.map(synchronizableToId)); + // this migration is no longer needed } } From 113ada4d79ff5ededb770e0fd7406969b884958e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20K=C3=B6nig?= Date: Thu, 17 Jul 2025 17:33:58 +0200 Subject: [PATCH 2/5] chore: remove not needed db schema upgrade --- .../runtime/src/DatabaseSchemaUpgrader.ts | 106 +----------------- 1 file changed, 3 insertions(+), 103 deletions(-) diff --git a/packages/runtime/src/DatabaseSchemaUpgrader.ts b/packages/runtime/src/DatabaseSchemaUpgrader.ts index b4d1853af..9c6956a63 100644 --- a/packages/runtime/src/DatabaseSchemaUpgrader.ts +++ b/packages/runtime/src/DatabaseSchemaUpgrader.ts @@ -1,9 +1,7 @@ -import { IDatabaseCollection } from "@js-soft/docdb-access-abstractions"; import { ILogger, ILoggerFactory } from "@js-soft/logging-abstractions"; import { Serializable, serialize, type, validate } from "@js-soft/ts-serval"; -import { ConsumptionController, LocalAttribute, LocalRequest } from "@nmshd/consumption"; -import { AccountController, DatawalletModification, DatawalletModificationCategory, DatawalletModificationType, TransportIds } from "@nmshd/transport"; -import _ from "lodash"; +import { ConsumptionController } from "@nmshd/consumption"; +import { AccountController } from "@nmshd/transport"; @type("RuntimeDatabaseSchemaMetadata") class RuntimeDatabaseSchemaMetadata extends Serializable { @@ -28,7 +26,7 @@ class RuntimeDatabaseSchemaMetadata extends Serializable { } export class DatabaseSchemaUpgrader { - private readonly CURRENT_DATABASE_SCHEMA_VERSION = 2; + private readonly CURRENT_DATABASE_SCHEMA_VERSION = 1; private readonly DATABASE_SCHEMA_QUERY = { id: RuntimeDatabaseSchemaMetadata.DATABASE_SCHEMA_ID }; public constructor( @@ -81,103 +79,5 @@ export const UPGRADE_LOGIC = Object.freeze { logger.info(`Upgrading database schema to version 1 for account '${accountController.identity.address.toString()}'`); - }, - // eslint-disable-next-line @typescript-eslint/naming-convention - 2: async (accountController, _, logger) => { - logger.info(`Upgrading database schema to version 2 for account '${accountController.identity.address.toString()}'`); - - if (!accountController.config.datawalletEnabled) return; - - const datawalletModifications = accountController["unpushedDatawalletModifications"] as IDatabaseCollection; - const datawalletVersion = accountController.config.supportedDatawalletVersion; - - const requestsCollection = await accountController.getSynchronizedCollection("Requests"); - const requestDocs = await requestsCollection.find({}); - - for (const requestDoc of requestDocs) { - logger.info(`Processing Request '${requestDoc.id}'`); - - let request: LocalRequest; - - try { - request = LocalRequest.from(requestDoc); - } catch (e) { - logger.error(`Failed to parse Request '${requestDoc.id}'`, e); - continue; - } - - const objectIdentifier = request.id; - - await datawalletModifications.create( - DatawalletModification.from({ - localId: await TransportIds.datawalletModification.generate(), - type: DatawalletModificationType.Create, - collection: "Requests", - objectIdentifier, - payloadCategory: DatawalletModificationCategory.TechnicalData, - payload: extractPayloadFromObject(request, request.technicalProperties), - datawalletVersion - }) - ); - - await datawalletModifications.create( - DatawalletModification.from({ - localId: await TransportIds.datawalletModification.generate(), - type: DatawalletModificationType.Create, - collection: "Requests", - objectIdentifier, - payloadCategory: DatawalletModificationCategory.Userdata, - payload: extractPayloadFromObject(request, request.userdataProperties), - datawalletVersion - }) - ); - - logger.info(`Successfully created datawallet modifications for Request '${requestDoc.id}'.`); - } - - const attributesCollection = await accountController.getSynchronizedCollection("Attributes"); - const attributeDocs = await attributesCollection.find({}); - - for (const attributeDoc of attributeDocs) { - logger.info(`Processing Attribute '${attributeDoc.id}'`); - - let attribute: LocalAttribute; - - try { - attribute = LocalAttribute.from(attributeDoc); - } catch (e) { - logger.error(`Failed to parse Attribute '${attributeDoc.id}'`, e); - continue; - } - - const technicalModificationPayload = extractPayloadFromObject(attribute, attribute.technicalProperties); - if (!("succeededBy" in technicalModificationPayload) && !("shareInfo" in technicalModificationPayload) && !("parentId" in technicalModificationPayload)) { - logger.info(`Attribute '${attributeDoc.id}' does not contain any new technical properties. Skipping.`); - continue; - } - - await datawalletModifications.create( - DatawalletModification.from({ - localId: await TransportIds.datawalletModification.generate(), - type: DatawalletModificationType.Update, - collection: "Attributes", - objectIdentifier: attribute.id, - payloadCategory: DatawalletModificationCategory.TechnicalData, - payload: technicalModificationPayload, - datawalletVersion - }) - ); - - logger.info(`Successfully created a datawallet modification for Attribute '${attributeDoc.id}'.`); - } - - await accountController.syncDatawallet(); } }); - -function extractPayloadFromObject(serializableObject: Serializable, properties: string[]): any { - const object = serializableObject.toJSON(); - const predicate = (value: any, key: string) => value !== undefined && properties.includes(key); - - return _.pickBy(object, predicate); -} From a380187647a576a19ace7c05c042995a55ba4fad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20K=C3=B6nig?= Date: Thu, 17 Jul 2025 17:34:13 +0200 Subject: [PATCH 3/5] chore: throw for to big db schema version --- packages/runtime/src/DatabaseSchemaUpgrader.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/runtime/src/DatabaseSchemaUpgrader.ts b/packages/runtime/src/DatabaseSchemaUpgrader.ts index 9c6956a63..f092a6122 100644 --- a/packages/runtime/src/DatabaseSchemaUpgrader.ts +++ b/packages/runtime/src/DatabaseSchemaUpgrader.ts @@ -38,6 +38,12 @@ export class DatabaseSchemaUpgrader { public async upgradeSchemaVersion(): Promise { let version = await this.getVersionFromDB(); + if (version > this.CURRENT_DATABASE_SCHEMA_VERSION) { + throw new Error( + `Database schema version ${version} is higher than the current version ${this.CURRENT_DATABASE_SCHEMA_VERSION}. This can happen if you started the application with unsupported data. Please reset the database.` + ); + } + while (version < this.CURRENT_DATABASE_SCHEMA_VERSION) { version++; From e8014b3ede9781edcc306c987b98b99b82ee15a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20K=C3=B6nig?= Date: Thu, 17 Jul 2025 17:50:46 +0200 Subject: [PATCH 4/5] chore: rm test --- .../test/misc/DatabaseSchemaUpgrader.test.ts | 205 +----------------- 1 file changed, 3 insertions(+), 202 deletions(-) diff --git a/packages/runtime/test/misc/DatabaseSchemaUpgrader.test.ts b/packages/runtime/test/misc/DatabaseSchemaUpgrader.test.ts index ed2a6d71e..0e57f3f8b 100644 --- a/packages/runtime/test/misc/DatabaseSchemaUpgrader.test.ts +++ b/packages/runtime/test/misc/DatabaseSchemaUpgrader.test.ts @@ -1,15 +1,11 @@ -import { IDatabaseCollection } from "@js-soft/docdb-access-abstractions"; import { LokiJsConnection } from "@js-soft/docdb-access-loki"; import { MongoDbConnection } from "@js-soft/docdb-access-mongo"; import { NodeLoggerFactory } from "@js-soft/node-logger"; import { EventEmitter2EventBus } from "@js-soft/ts-utils"; -import { ConsumptionController, LocalAttribute, LocalRequest } from "@nmshd/consumption"; -import { DisplayName, IdentityAttribute } from "@nmshd/content"; -import { CoreAddress, CoreDate, CoreId } from "@nmshd/core-types"; +import { ConsumptionController } from "@nmshd/consumption"; import { AccountController, Transport } from "@nmshd/transport"; -import { LocalRequestStatus } from "../../src"; -import { DatabaseSchemaUpgrader, UPGRADE_LOGIC } from "../../src/DatabaseSchemaUpgrader"; -import { RuntimeServiceProvider, TestRequestItem } from "../lib"; +import { DatabaseSchemaUpgrader } from "../../src/DatabaseSchemaUpgrader"; +import { RuntimeServiceProvider } from "../lib"; const loggerFactory = new NodeLoggerFactory({ appenders: { @@ -76,199 +72,4 @@ describe("DatabaseSchemaUpgrader", () => { expect(doc).toBeDefined(); expect(doc.version).toBeGreaterThan(0); }); - - describe("migration from version 1 to 2", () => { - let unpushedDatawalletModifications: IDatabaseCollection; - const v2UpgradeFunction = UPGRADE_LOGIC[2]!; - - beforeAll(() => { - unpushedDatawalletModifications = accountController["unpushedDatawalletModifications"] as IDatabaseCollection; - }); - - beforeEach(async () => { - jest.clearAllMocks(); - - const deleteAllDocuments = async (collection: IDatabaseCollection, identifier = "id") => { - const documents = await collection.find(); - for (const document of documents) await collection.delete({ [identifier]: document[identifier] }); - }; - - await deleteAllDocuments(unpushedDatawalletModifications, "localId"); - await deleteAllDocuments(await accountController.db.getCollection("Requests")); - await deleteAllDocuments(await accountController.db.getCollection("Attributes")); - - jest.spyOn(accountController, "syncDatawallet").mockImplementation(); - }); - - test("should not create datawallet modifications or sync when no requests or attributes exist", async () => { - await v2UpgradeFunction(accountController, undefined as any as ConsumptionController, testLogger); - - expect(accountController.syncDatawallet).toHaveBeenCalled(); - expect(await unpushedDatawalletModifications.count()).toBe(0); - }); - - test("should create datawallet modifications and sync when a request exist", async () => { - const requestsCollection = await accountController.db.getCollection("Requests"); - await requestsCollection.create( - LocalRequest.from({ - id: CoreId.from("REQ123"), - content: { items: [TestRequestItem.from({ mustBeAccepted: false })] }, - isOwn: true, - createdAt: CoreDate.utc(), - peer: CoreAddress.from(""), - status: LocalRequestStatus.Draft, - statusLog: [] - }) - ); - - await v2UpgradeFunction(accountController, undefined as any as ConsumptionController, testLogger); - - expect(accountController.syncDatawallet).toHaveBeenCalled(); - expect(await unpushedDatawalletModifications.count()).toBe(2); - - const modifications = await unpushedDatawalletModifications.find({}); - expect(modifications).toHaveLength(2); - - const technicalDataModification = modifications[0]; - expect(technicalDataModification.payloadCategory).toBe("TechnicalData"); - expect(technicalDataModification.type).toBe("Create"); - expect(Object.keys(technicalDataModification.payload)).toStrictEqual(["@type", "createdAt", "isOwn", "peer", "status", "statusLog"]); - - const userDataModification = modifications[1]; - expect(userDataModification.payloadCategory).toBe("Userdata"); - expect(userDataModification.type).toBe("Create"); - expect(Object.keys(userDataModification.payload)).toStrictEqual(["content"]); - }); - - test("should create datawallet modifications and sync when two requests exist", async () => { - const requestsCollection = await accountController.db.getCollection("Requests"); - await requestsCollection.create( - LocalRequest.from({ - id: CoreId.from("REQ123"), - content: { items: [TestRequestItem.from({ mustBeAccepted: false })] }, - isOwn: true, - createdAt: CoreDate.utc(), - peer: CoreAddress.from(""), - status: LocalRequestStatus.Draft, - statusLog: [] - }) - ); - - await requestsCollection.create( - LocalRequest.from({ - id: CoreId.from("REQ456"), - content: { items: [TestRequestItem.from({ mustBeAccepted: false })] }, - isOwn: true, - createdAt: CoreDate.utc(), - peer: CoreAddress.from(""), - status: LocalRequestStatus.Draft, - statusLog: [] - }) - ); - - await v2UpgradeFunction(accountController, undefined as any as ConsumptionController, testLogger); - - expect(accountController.syncDatawallet).toHaveBeenCalled(); - expect(await unpushedDatawalletModifications.count()).toBe(4); - }); - - test("should create a datawallet modification and sync when an attribute exists", async () => { - const attributesCollection = await accountController.db.getCollection("Attributes"); - await attributesCollection.create( - LocalAttribute.from({ - id: CoreId.from("ATT123"), - content: IdentityAttribute.from({ - owner: CoreAddress.from(""), - value: DisplayName.from("Test") - }), - createdAt: CoreDate.utc(), - shareInfo: { - peer: CoreAddress.from(""), - requestReference: CoreId.from("REQ123") - } - }) - ); - - await v2UpgradeFunction(accountController, undefined as any as ConsumptionController, testLogger); - - expect(accountController.syncDatawallet).toHaveBeenCalled(); - - const modifications = await unpushedDatawalletModifications.find({}); - expect(modifications).toHaveLength(1); - - const modification = modifications[0]; - - expect(modification.payloadCategory).toBe("TechnicalData"); - expect(modification.type).toBe("Update"); - expect(Object.keys(modification.payload)).toStrictEqual(["@type", "createdAt", "shareInfo"]); - }); - - test("should create no datawallet modification when an attribute exists but has no new technical properties", async () => { - const attributesCollection = await accountController.db.getCollection("Attributes"); - await attributesCollection.create( - LocalAttribute.from({ - id: CoreId.from("ATT123"), - content: IdentityAttribute.from({ - owner: CoreAddress.from(""), - value: DisplayName.from("Test") - }), - createdAt: CoreDate.utc() - }) - ); - - await v2UpgradeFunction(accountController, undefined as any as ConsumptionController, testLogger); - - expect(accountController.syncDatawallet).toHaveBeenCalled(); - expect(await unpushedDatawalletModifications.count()).toBe(0); - }); - - test("should create a datawallet modification and sync when two attributes exists", async () => { - const attributesCollection = await accountController.db.getCollection("Attributes"); - await attributesCollection.create( - LocalAttribute.from({ - id: CoreId.from("ATT123"), - content: IdentityAttribute.from({ - owner: CoreAddress.from(""), - value: DisplayName.from("Test") - }), - createdAt: CoreDate.utc(), - shareInfo: { - peer: CoreAddress.from(""), - requestReference: CoreId.from("REQ123") - } - }) - ); - - await attributesCollection.create( - LocalAttribute.from({ - id: CoreId.from("ATT456"), - content: IdentityAttribute.from({ - owner: CoreAddress.from(""), - value: DisplayName.from("Test") - }), - createdAt: CoreDate.utc(), - shareInfo: { - peer: CoreAddress.from(""), - requestReference: CoreId.from("REQ123") - } - }) - ); - - await v2UpgradeFunction(accountController, undefined as any as ConsumptionController, testLogger); - - expect(accountController.syncDatawallet).toHaveBeenCalled(); - expect(await unpushedDatawalletModifications.count()).toBe(2); - }); - - test("should not create datawallet modifications or sync when the datawallet is disabled", async () => { - accountController.config.datawalletEnabled = false; - - await v2UpgradeFunction(accountController, undefined as any as ConsumptionController, testLogger); - - expect(accountController.syncDatawallet).not.toHaveBeenCalled(); - expect(await unpushedDatawalletModifications.count()).toBe(0); - - accountController.config.datawalletEnabled = true; - }); - }); }); From 6ea59a22c21d2e703029aa51a910cd778ef6d8d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20K=C3=B6nig?= Date: Fri, 18 Jul 2025 07:47:21 +0200 Subject: [PATCH 5/5] chore: update comment --- .../src/modules/sync/migrations/DeviceMigrations.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/transport/src/modules/sync/migrations/DeviceMigrations.ts b/packages/transport/src/modules/sync/migrations/DeviceMigrations.ts index 9794a7086..c6d8ae1c9 100644 --- a/packages/transport/src/modules/sync/migrations/DeviceMigrations.ts +++ b/packages/transport/src/modules/sync/migrations/DeviceMigrations.ts @@ -3,7 +3,8 @@ import { AccountController } from "../../accounts/AccountController"; export class DeviceMigrations { public constructor(private readonly accountController: AccountController) {} - public async v1(): Promise { - // this migration is no longer needed + public v1(): Promise { + // no upgrade steps necessary for v1 + return Promise.resolve(); } }