From f37871cca9eb3d1c273f2e77469543495f263756 Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Thu, 25 Apr 2024 16:48:59 +0700 Subject: [PATCH 01/34] Remove mongodb dependency from collaborator Signed-off-by: Alexander Onnikov --- dev/docker-compose.yaml | 2 - pods/collaborator/Dockerfile | 2 +- pods/collaborator/package.json | 2 +- server/collaborator/Dockerfile | 9 --- server/collaborator/package.json | 12 +--- server/collaborator/src/server.ts | 2 +- server/collaborator/src/storage/platform.ts | 65 ++------------------- tests/docker-compose.yaml | 2 - 8 files changed, 10 insertions(+), 86 deletions(-) delete mode 100644 server/collaborator/Dockerfile diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml index c2f6254e2dd..c8c7d86e595 100644 --- a/dev/docker-compose.yaml +++ b/dev/docker-compose.yaml @@ -89,7 +89,6 @@ services: collaborator: image: hardcoreeng/collaborator links: - - mongodb - minio - transactor ports: @@ -100,7 +99,6 @@ services: - ACCOUNTS_URL=http://account:3000 - TRANSACTOR_URL=ws://transactor:3333 - UPLOAD_URL=/files - - MONGO_URL=mongodb://mongodb:27017 - STORAGE_CONFIG=${STORAGE_CONFIG} restart: unless-stopped # tracker-front: diff --git a/pods/collaborator/Dockerfile b/pods/collaborator/Dockerfile index 87d78fc24b8..ae42281a03b 100644 --- a/pods/collaborator/Dockerfile +++ b/pods/collaborator/Dockerfile @@ -2,7 +2,7 @@ FROM node:20 WORKDIR /usr/src/app -RUN npm install --ignore-scripts=false --verbose bufferutil utf-8-validate @mongodb-js/zstd --unsafe-perm +RUN npm install --ignore-scripts=false --verbose bufferutil utf-8-validate --unsafe-perm COPY bundle/bundle.js ./ diff --git a/pods/collaborator/package.json b/pods/collaborator/package.json index a06dbb43259..7d6a82eaacc 100644 --- a/pods/collaborator/package.json +++ b/pods/collaborator/package.json @@ -17,7 +17,7 @@ "docker:build": "../../common/scripts/docker_build.sh hardcoreeng/collaborator", "docker:staging": "../../common/scripts/docker_tag.sh hardcoreeng/collaborator staging", "docker:push": "../../common/scripts/docker_tag.sh hardcoreeng/collaborator", - "run-local": "cross-env MONGO_URL=mongodb://localhost:27017 TRANSACTOR_URL=ws://localhost:3333 SECRET=secret MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin ts-node src/__start.ts", + "run-local": "cross-env TRANSACTOR_URL=ws://localhost:3333 SECRET=secret MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin ts-node src/__start.ts", "format": "format src", "test": "jest --passWithNoTests --silent --forceExit", "_phase:build": "compile transpile src", diff --git a/server/collaborator/Dockerfile b/server/collaborator/Dockerfile deleted file mode 100644 index 6bc7f5c41ba..00000000000 --- a/server/collaborator/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ - -FROM node:20-alpine - -WORKDIR /usr/src/app - -COPY bundle/bundle.js ./ - -EXPOSE 3078 -CMD [ "node", "bundle.js" ] diff --git a/server/collaborator/package.json b/server/collaborator/package.json index 08a8a7de19f..6210b510cbf 100644 --- a/server/collaborator/package.json +++ b/server/collaborator/package.json @@ -14,16 +14,11 @@ "scripts": { "build": "compile", "build:watch": "compile", - "_phase:bundle": "rushx bundle", - "_phase:docker-build": "rushx docker:build", - "_phase:docker-staging": "rushx docker:staging", - "bundle": "mkdir -p bundle && esbuild src/__start.ts --bundle --platform=node > bundle/bundle.js", - "docker:build": "../../common/scripts/docker_build.sh hardcoreeng/collaborator", - "docker:staging": "../../common/scripts/docker_tag.sh hardcoreeng/collaborator staging", - "docker:push": "../../common/scripts/docker_tag.sh hardcoreeng/collaborator", - "run-local": "cross-env MONGO_URL=mongodb://localhost:27017 TRANSACTOR_URL=ws://localhost:3333 SECRET=secret MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin ts-node src/__start.ts", "format": "format src", "test": "jest --passWithNoTests --silent", + "_phase:bundle": "rushx bundle", + "bundle": "mkdir -p bundle && esbuild src/__start.ts --bundle --platform=node > bundle/bundle.js", + "run-local": "cross-env TRANSACTOR_URL=ws://localhost:3333 SECRET=secret MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin ts-node src/__start.ts", "_phase:build": "compile transpile src", "_phase:test": "jest --passWithNoTests --silent", "_phase:format": "format src", @@ -73,7 +68,6 @@ "@hocuspocus/transformer": "^2.9.0", "@tiptap/core": "^2.2.4", "@tiptap/html": "^2.2.4", - "mongodb": "^6.3.0", "yjs": "^13.5.52", "y-prosemirror": "^1.2.1", "express": "^4.18.3", diff --git a/server/collaborator/src/server.ts b/server/collaborator/src/server.ts index 2cfdcb428e7..3d755c89da4 100644 --- a/server/collaborator/src/server.ts +++ b/server/collaborator/src/server.ts @@ -133,7 +133,7 @@ export async function start ( }), new StorageExtension({ ctx: extensionsCtx.newChild('storage', {}), - adapter: new PlatformStorageAdapter({ minio }, mongo, transformer) + adapter: new PlatformStorageAdapter({ minio }, transformer) }) ] }) diff --git a/server/collaborator/src/storage/platform.ts b/server/collaborator/src/storage/platform.ts index 54f0474900c..6bcf286d5a1 100644 --- a/server/collaborator/src/storage/platform.ts +++ b/server/collaborator/src/storage/platform.ts @@ -27,16 +27,12 @@ import { } from '@hcengineering/collaborator-client' import core, { CollaborativeDoc, - Doc, MeasureContext, TxOperations, collaborativeDocWithLastVersion, - toWorkspaceString } from '@hcengineering/core' import { StorageAdapter } from '@hcengineering/server-core' -import { areEqualMarkups } from '@hcengineering/text' import { Transformer } from '@hocuspocus/transformer' -import { MongoClient } from 'mongodb' import { Doc as YDoc } from 'yjs' import { Context } from '../context' @@ -47,7 +43,6 @@ export type StorageAdapters = Record export class PlatformStorageAdapter implements CollabStorageAdapter { constructor ( private readonly adapters: StorageAdapters, - private readonly mongodb: MongoClient, private readonly transformer: Transformer ) {} @@ -84,27 +79,6 @@ export class PlatformStorageAdapter implements CollabStorageAdapter { } } - // finally try to load from the platform - const { platformDocumentId } = context - if (platformDocumentId !== undefined) { - ctx.info('load document platform content', { documentId, platformDocumentId }) - const ydoc = await ctx.with('load-from-platform', {}, async (ctx) => { - try { - return await this.loadDocumentFromPlatform(ctx, platformDocumentId, context) - } catch (err) { - ctx.error('failed to load platform document', { documentId, platformDocumentId, error: err }) - } - }) - - // if document was loaded from the initial content or storage we need to save - // it to ensure the next time we load it from the ydoc document - if (ydoc !== undefined) { - ctx.info('save document content', { documentId, platformDocumentId }) - await this.saveDocumentToStorage(ctx, documentId, ydoc, context) - return ydoc - } - } - // nothing found return undefined } catch (err) { @@ -120,6 +94,8 @@ export class PlatformStorageAdapter implements CollabStorageAdapter { }) try { + // TODO even though there were changes, the document content could be the same + // for example, when user added several words and then removed them let snapshot: YDocVersion | undefined try { ctx.info('take document snapshot', { documentId }) @@ -217,31 +193,6 @@ export class PlatformStorageAdapter implements CollabStorageAdapter { return yDocVersion } - async loadDocumentFromPlatform ( - ctx: MeasureContext, - platformDocumentId: PlatformDocumentId, - context: Context - ): Promise { - const { mongodb, transformer } = this - const { workspaceId } = context - const { objectDomain, objectId, objectAttr } = parsePlatformDocumentId(platformDocumentId) - - const doc = await ctx.with('query', {}, async () => { - const db = mongodb.db(toWorkspaceString(workspaceId)) - return await db.collection(objectDomain).findOne({ _id: objectId }, { projection: { [objectAttr]: 1 } }) - }) - - const content = doc !== null && objectAttr in doc ? ((doc as any)[objectAttr] as string) : '' - if (content.startsWith('{') && content.endsWith('}')) { - return await ctx.with('transform', {}, () => { - return transformer.toYdoc(content, objectAttr) - }) - } - - // the content does not seem to be an HTML document - return undefined - } - async saveDocumentToPlatform ( ctx: MeasureContext, client: Omit, @@ -278,16 +229,8 @@ export class PlatformStorageAdapter implements CollabStorageAdapter { await ctx.with('update', {}, async () => { await client.diffUpdate(current, { [objectAttr]: newCollaborativeDoc }) }) - } else if (hierarchy.isDerived(attribute.type._class, core.class.TypeCollaborativeMarkup)) { - // TODO a temporary solution while we are keeping Markup in Mongo - const content = await ctx.with('transform', {}, () => { - return this.transformer.fromYdoc(document, objectAttr) - }) - if (!areEqualMarkups(content, (current as any)[objectAttr])) { - await ctx.with('update', {}, async () => { - await client.diffUpdate(current, { [objectAttr]: content }) - }) - } + } else { + ctx.error('unsupported attribute type', { documentName, objectClass, objectAttr }) } } } diff --git a/tests/docker-compose.yaml b/tests/docker-compose.yaml index a8f44bd399b..c14ad998448 100644 --- a/tests/docker-compose.yaml +++ b/tests/docker-compose.yaml @@ -106,7 +106,6 @@ services: collaborator: image: hardcoreeng/collaborator links: - - mongodb - minio - transactor ports: @@ -117,7 +116,6 @@ services: - ACCOUNTS_URL=http://account:3003 - TRANSACTOR_URL=ws://transactor:3334 - UPLOAD_URL=/files - - MONGO_URL=mongodb://mongodb:27018 - STORAGE_CONFIG=${STORAGE_CONFIG} restart: unless-stopped rekoni: From 18f8c8ea0a7ae33eec121ebbec10b697ecf1f54d Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Thu, 25 Apr 2024 22:46:40 +0700 Subject: [PATCH 02/34] Remove CollaborativeMarkup data type Signed-off-by: Alexander Onnikov --- dev/tool/src/markup.ts | 5 +---- models/activity/src/migration.ts | 4 +--- models/contact/src/index.ts | 7 ++++--- models/core/src/core.ts | 4 ---- models/core/src/index.ts | 2 -- models/lead/src/types.ts | 9 +++++---- models/recruit/src/types.ts | 7 ++++--- models/text-editor/src/migration.ts | 10 ++-------- models/tracker/src/types.ts | 11 ++++++----- models/view/src/index.ts | 13 ------------- packages/core/src/component.ts | 1 - packages/model/src/dsl.ts | 7 ------- packages/presentation/src/utils.ts | 3 --- .../activity-resources/src/activityMessagesUtils.ts | 1 - plugins/activity-resources/src/utils.ts | 5 +---- plugins/lead/src/index.ts | 4 ++-- plugins/recruit/src/types.ts | 4 ++-- plugins/tracker/src/index.ts | 3 ++- server-plugins/activity-resources/src/references.ts | 2 +- server/collaborator/src/account.ts | 6 +++--- server/core/src/indexer/fulltextPush.ts | 5 +---- server/core/src/indexer/summary.ts | 2 +- 22 files changed, 36 insertions(+), 79 deletions(-) diff --git a/dev/tool/src/markup.ts b/dev/tool/src/markup.ts index b1f512a3419..7053be2ede3 100644 --- a/dev/tool/src/markup.ts +++ b/dev/tool/src/markup.ts @@ -38,10 +38,7 @@ export async function fixJsonMarkup ( const attributes = hierarchy.getAllAttributes(_class) const filtered = Array.from(attributes.values()).filter((attribute) => { - return ( - hierarchy.isDerived(attribute.type._class, core.class.TypeMarkup) || - hierarchy.isDerived(attribute.type._class, core.class.TypeCollaborativeMarkup) - ) + return hierarchy.isDerived(attribute.type._class, core.class.TypeMarkup) }) if (filtered.length === 0) continue diff --git a/models/activity/src/migration.ts b/models/activity/src/migration.ts index bd71b840286..3c75a15894f 100644 --- a/models/activity/src/migration.ts +++ b/models/activity/src/migration.ts @@ -45,9 +45,7 @@ async function migrateMarkup (client: MigrationClient): Promise { DOMAIN_ACTIVITY, { _class: activity.class.DocUpdateMessage, - 'attributeUpdates.attrClass': { - $in: [core.class.TypeMarkup, core.class.TypeCollaborativeMarkup] - } + 'attributeUpdates.attrClass': core.class.TypeMarkup }, { projection: { diff --git a/models/contact/src/index.ts b/models/contact/src/index.ts index 2f573809f23..1c2270b9f72 100644 --- a/models/contact/src/index.ts +++ b/models/contact/src/index.ts @@ -42,6 +42,7 @@ import { type Markup, type Ref, type Timestamp + type CollaborativeDoc } from '@hcengineering/core' import { Collection, @@ -53,7 +54,7 @@ import { ReadOnly, TypeBlob, TypeBoolean, - TypeCollaborativeMarkup, + TypeCollaborativeDoc, TypeDate, TypeRecord, TypeRef, @@ -172,9 +173,9 @@ export class TMember extends TAttachedDoc implements Member { @Model(contact.class.Organization, contact.class.Contact) @UX(contact.string.Organization, contact.icon.Company, 'ORG', 'name', undefined, contact.string.Organizations) export class TOrganization extends TContact implements Organization { - @Prop(TypeCollaborativeMarkup(), core.string.Description) + @Prop(TypeCollaborativeDoc(), core.string.Description) @Index(IndexKind.FullText) - description?: Markup + description?: CollaborativeDoc @Prop(Collection(contact.class.Member), contact.string.Members) members!: number diff --git a/models/core/src/core.ts b/models/core/src/core.ts index 317a7be3416..2ac14199cfe 100644 --- a/models/core/src/core.ts +++ b/models/core/src/core.ts @@ -232,10 +232,6 @@ export class TTypeNumber extends TType {} @Model(core.class.TypeMarkup, core.class.Type) export class TTypeMarkup extends TType {} -@UX(core.string.Collaborative) -@Model(core.class.TypeCollaborativeMarkup, core.class.Type) -export class TTypeCollaborativeMarkup extends TType {} - @UX(core.string.Ref) @Model(core.class.RefTo, core.class.Type) export class TRefTo extends TType implements RefTo { diff --git a/models/core/src/index.ts b/models/core/src/index.ts index 2d0105c52e6..a290b2b33a2 100644 --- a/models/core/src/index.ts +++ b/models/core/src/index.ts @@ -64,7 +64,6 @@ import { TTypeBoolean, TTypeCollaborativeDoc, TTypeCollaborativeDocVersion, - TTypeCollaborativeMarkup, TTypeDate, TTypeHyperlink, TTypeIntlString, @@ -143,7 +142,6 @@ export function createModel (builder: Builder): void { TTypeMarkup, TTypeCollaborativeDoc, TTypeCollaborativeDocVersion, - TTypeCollaborativeMarkup, TArrOf, TRefTo, TTypeDate, diff --git a/models/lead/src/types.ts b/models/lead/src/types.ts index 53fce71c180..f48f59657c1 100644 --- a/models/lead/src/types.ts +++ b/models/lead/src/types.ts @@ -21,7 +21,8 @@ import { type RolesAssignment, type Ref, type Status, - type Timestamp + type Timestamp, + CollaborativeDoc } from '@hcengineering/core' import { type Customer, type Funnel, type Lead } from '@hcengineering/lead' import { @@ -31,7 +32,7 @@ import { Model, Prop, ReadOnly, - TypeCollaborativeMarkup, + TypeCollaborativeDoc, TypeDate, TypeMarkup, TypeRef, @@ -94,9 +95,9 @@ export class TCustomer extends TContact implements Customer { @Prop(Collection(lead.class.Lead), lead.string.Leads) leads?: number - @Prop(TypeCollaborativeMarkup(), core.string.Description) + @Prop(TypeCollaborativeDoc(), core.string.Description) @Index(IndexKind.FullText) - description!: string + description!: CollaborativeDoc } @Mixin(lead.mixin.DefaultFunnelTypeData, lead.class.Funnel) diff --git a/models/recruit/src/types.ts b/models/recruit/src/types.ts index 3ed58cb3043..925cb8aa0f6 100644 --- a/models/recruit/src/types.ts +++ b/models/recruit/src/types.ts @@ -17,6 +17,7 @@ import type { Employee, Organization } from '@hcengineering/contact' import { Account, IndexKind, + type CollaborativeDoc, type Domain, type Markup, type Ref, @@ -34,7 +35,7 @@ import { Prop, ReadOnly, TypeBoolean, - TypeCollaborativeMarkup, + TypeCollaborativeDoc, TypeDate, TypeMarkup, TypeRef, @@ -63,9 +64,9 @@ import recruit from './plugin' @Model(recruit.class.Vacancy, task.class.Project) @UX(recruit.string.Vacancy, recruit.icon.Vacancy, 'VCN', 'name', undefined, recruit.string.Vacancies) export class TVacancy extends TProject implements Vacancy { - @Prop(TypeCollaborativeMarkup(), recruit.string.FullDescription) + @Prop(TypeCollaborativeDoc(), recruit.string.FullDescription) @Index(IndexKind.FullText) - fullDescription?: string + fullDescription?: CollaborativeDoc @Prop(Collection(attachment.class.Attachment), attachment.string.Attachments, { shortLabel: attachment.string.Files }) attachments?: number diff --git a/models/text-editor/src/migration.ts b/models/text-editor/src/migration.ts index ad667d5f994..0d0ada5e7ce 100644 --- a/models/text-editor/src/migration.ts +++ b/models/text-editor/src/migration.ts @@ -34,10 +34,7 @@ async function migrateMarkup (client: MigrationClient): Promise { const attributes = hierarchy.getAllAttributes(_class) const filtered = Array.from(attributes.values()).filter((attribute) => { - return ( - hierarchy.isDerived(attribute.type._class, core.class.TypeMarkup) || - hierarchy.isDerived(attribute.type._class, core.class.TypeCollaborativeMarkup) - ) + return hierarchy.isDerived(attribute.type._class, core.class.TypeMarkup) }) if (filtered.length === 0) continue @@ -106,10 +103,7 @@ async function fixMigrateMarkup (client: MigrationClient): Promise { const attributes = hierarchy.getAllAttributes(_class) const filtered = Array.from(attributes.values()).filter((attribute) => { - return ( - hierarchy.isDerived(attribute.type._class, core.class.TypeMarkup) || - hierarchy.isDerived(attribute.type._class, core.class.TypeCollaborativeMarkup) - ) + return hierarchy.isDerived(attribute.type._class, core.class.TypeMarkup) }) if (filtered.length === 0) continue diff --git a/models/tracker/src/types.ts b/models/tracker/src/types.ts index 6a9f57387c0..cb63cb21b89 100644 --- a/models/tracker/src/types.ts +++ b/models/tracker/src/types.ts @@ -18,8 +18,9 @@ import contact, { type Employee, type Person } from '@hcengineering/contact' import { DOMAIN_MODEL, DateRangeMode, - type Domain, IndexKind, + type CollaborativeDoc, + type Domain, type Markup, type Ref, type RelatedDocument, @@ -38,7 +39,7 @@ import { Model, Prop, ReadOnly, - TypeCollaborativeMarkup, + TypeCollaborativeDoc, TypeDate, TypeMarkup, TypeNumber, @@ -180,9 +181,9 @@ export class TIssue extends TTask implements Issue { @Index(IndexKind.FullText) title!: string - @Prop(TypeCollaborativeMarkup(), tracker.string.Description) + @Prop(TypeCollaborativeDoc(), tracker.string.Description) @Index(IndexKind.FullText) - description!: Markup + description!: CollaborativeDoc @Prop(TypeRef(tracker.class.IssueStatus), tracker.string.Status, { _id: tracker.attribute.IssueStatus, @@ -270,7 +271,7 @@ export class TIssueTemplate extends TDoc implements IssueTemplate { @Index(IndexKind.FullText) title!: string - @Prop(TypeCollaborativeMarkup(), tracker.string.Description) + @Prop(TypeMarkup(), tracker.string.Description) @Index(IndexKind.FullText) description!: Markup diff --git a/models/view/src/index.ts b/models/view/src/index.ts index c939c22683e..6c5b96674f1 100644 --- a/models/view/src/index.ts +++ b/models/view/src/index.ts @@ -483,19 +483,6 @@ export function createModel (builder: Builder): void { editor: view.component.HTMLEditor }) - classPresenter( - builder, - core.class.TypeCollaborativeMarkup, - view.component.MarkupPresenter, - undefined, - undefined, - view.component.MarkupDiffPresenter - ) - - builder.mixin(core.class.TypeCollaborativeMarkup, core.class.Class, view.mixin.InlineAttributEditor, { - editor: view.component.CollaborativeHTMLEditor - }) - builder.mixin(core.class.TypeCollaborativeDoc, core.class.Class, view.mixin.InlineAttributEditor, { editor: view.component.CollaborativeDocEditor }) diff --git a/packages/core/src/component.ts b/packages/core/src/component.ts index c7935299106..a16e8232de2 100644 --- a/packages/core/src/component.ts +++ b/packages/core/src/component.ts @@ -123,7 +123,6 @@ export default plugin(coreId, { TypeDate: '' as Ref>>, TypeCollaborativeDoc: '' as Ref>>, TypeCollaborativeDocVersion: '' as Ref>>, - TypeCollaborativeMarkup: '' as Ref>>, RefTo: '' as Ref>>, ArrOf: '' as Ref>>, Enum: '' as Ref>, diff --git a/packages/model/src/dsl.ts b/packages/model/src/dsl.ts index e81a867a664..b68cdcf3381 100644 --- a/packages/model/src/dsl.ts +++ b/packages/model/src/dsl.ts @@ -412,13 +412,6 @@ export function TypeMarkup (): Type { return { _class: core.class.TypeMarkup, label: core.string.Markup } } -/** - * @public - */ -export function TypeCollaborativeMarkup (): Type { - return { _class: core.class.TypeCollaborativeMarkup, label: core.string.Collaborative } -} - /** * @public */ diff --git a/packages/presentation/src/utils.ts b/packages/presentation/src/utils.ts index 912d2aa0ae3..ef8b5ff35c3 100644 --- a/packages/presentation/src/utils.ts +++ b/packages/presentation/src/utils.ts @@ -474,9 +474,6 @@ export function getAttributePresenterClass ( if (hierarchy.isDerived(attrClass, core.class.TypeMarkup)) { category = 'inplace' } - if (hierarchy.isDerived(attrClass, core.class.TypeCollaborativeMarkup)) { - category = 'inplace' - } if (hierarchy.isDerived(attrClass, core.class.TypeCollaborativeDoc)) { category = 'inplace' } diff --git a/plugins/activity-resources/src/activityMessagesUtils.ts b/plugins/activity-resources/src/activityMessagesUtils.ts index 2c5d7127ea8..0d510b00459 100644 --- a/plugins/activity-resources/src/activityMessagesUtils.ts +++ b/plugins/activity-resources/src/activityMessagesUtils.ts @@ -58,7 +58,6 @@ const valueTypes: ReadonlyArray>> = [ core.class.TypeNumber, core.class.TypeDate, core.class.TypeMarkup, - core.class.TypeCollaborativeMarkup, core.class.TypeHyperlink ] diff --git a/plugins/activity-resources/src/utils.ts b/plugins/activity-resources/src/utils.ts index 2e0a262325e..5cfbf4f320a 100644 --- a/plugins/activity-resources/src/utils.ts +++ b/plugins/activity-resources/src/utils.ts @@ -136,10 +136,7 @@ export function getIsTextType (attributeModel?: AttributeModel): boolean { return false } - return ( - attributeModel.attribute?.type?._class === core.class.TypeMarkup || - attributeModel.attribute?.type?._class === core.class.TypeCollaborativeMarkup - ) + return attributeModel.attribute?.type?._class === core.class.TypeMarkup } const groupMessagesThresholdMs = 15 * 60 * 1000 diff --git a/plugins/lead/src/index.ts b/plugins/lead/src/index.ts index 98dcb960cc1..c83bdbb8984 100644 --- a/plugins/lead/src/index.ts +++ b/plugins/lead/src/index.ts @@ -15,7 +15,7 @@ // import type { Contact } from '@hcengineering/contact' -import type { Attribute, Class, Doc, Markup, Ref, Status, Timestamp } from '@hcengineering/core' +import type { Attribute, Class, CollaborativeDoc, Doc, Markup, Ref, Status, Timestamp } from '@hcengineering/core' import { Mixin } from '@hcengineering/core' import type { Asset, IntlString, Plugin } from '@hcengineering/platform' import { plugin } from '@hcengineering/platform' @@ -36,7 +36,7 @@ export interface Funnel extends Project { export interface Customer extends Contact { leads?: number - description: string + description: CollaborativeDoc } /** diff --git a/plugins/recruit/src/types.ts b/plugins/recruit/src/types.ts index 471d2d872f7..bff863b5590 100644 --- a/plugins/recruit/src/types.ts +++ b/plugins/recruit/src/types.ts @@ -15,13 +15,13 @@ import { Event } from '@hcengineering/calendar' import type { Channel, Organization, Person } from '@hcengineering/contact' -import type { AttachedData, AttachedDoc, Markup, Ref, Status, Timestamp } from '@hcengineering/core' +import type { AttachedData, AttachedDoc, CollaborativeDoc, Markup, Ref, Status, Timestamp } from '@hcengineering/core' import { TagReference } from '@hcengineering/tags' import type { Project, Task } from '@hcengineering/task' /** @public */ export interface Vacancy extends Project { - fullDescription?: string + fullDescription?: CollaborativeDoc attachments?: number dueTo?: Timestamp location?: string diff --git a/plugins/tracker/src/index.ts b/plugins/tracker/src/index.ts index 17ddc4fdf97..ffe6666cefc 100644 --- a/plugins/tracker/src/index.ts +++ b/plugins/tracker/src/index.ts @@ -18,6 +18,7 @@ import { AttachedDoc, Attribute, Class, + CollaborativeDoc, CollectionSize, Data, Doc, @@ -181,7 +182,7 @@ export interface Milestone extends Doc { export interface Issue extends Task { attachedTo: Ref title: string - description: Markup + description: CollaborativeDoc status: Ref priority: IssuePriority diff --git a/server-plugins/activity-resources/src/references.ts b/server-plugins/activity-resources/src/references.ts index 01876621a41..7a8fadbf602 100644 --- a/server-plugins/activity-resources/src/references.ts +++ b/server-plugins/activity-resources/src/references.ts @@ -318,7 +318,7 @@ async function getMessageNotifyResult ( } function isMarkupType (type: Ref>>): boolean { - return type === core.class.TypeMarkup || type === core.class.TypeCollaborativeMarkup + return type === core.class.TypeMarkup } function isCollaborativeType (type: Ref>>): boolean { diff --git a/server/collaborator/src/account.ts b/server/collaborator/src/account.ts index ee350e38970..33f525921ab 100644 --- a/server/collaborator/src/account.ts +++ b/server/collaborator/src/account.ts @@ -13,10 +13,10 @@ // limitations under the License. // -import { WorkspaceLoginInfo } from '@hcengineering/account' +import { ClientWorkspaceInfo } from '@hcengineering/account' import config from './config' -export async function getWorkspaceInfo (token: string): Promise { +export async function getWorkspaceInfo (token: string): Promise { const accountsUrl = config.AccountsUrl const workspaceInfo = await ( await fetch(accountsUrl, { @@ -32,5 +32,5 @@ export async function getWorkspaceInfo (token: string): Promise Date: Thu, 2 May 2024 17:05:34 +0700 Subject: [PATCH 03/34] Rename getCollaborativeDocId to makeCollaborativeDoc Signed-off-by: Alexander Onnikov --- dev/tool/src/markup.ts | 4 ++-- models/contact/src/index.ts | 3 +-- packages/core/src/collaboration.ts | 10 +++------- .../src/components/CollaborativeAttributeBox.svelte | 5 ++--- plugins/bitrix/src/hr.ts | 12 ++++++++---- plugins/document-resources/src/utils.ts | 8 +++----- .../src/components/CreateIssue.svelte | 1 + 7 files changed, 20 insertions(+), 23 deletions(-) diff --git a/dev/tool/src/markup.ts b/dev/tool/src/markup.ts index 7053be2ede3..75c64b3bd27 100644 --- a/dev/tool/src/markup.ts +++ b/dev/tool/src/markup.ts @@ -7,7 +7,7 @@ import core, { type MeasureContext, type Ref, type WorkspaceId, - getCollaborativeDocId + makeCollaborativeDoc } from '@hcengineering/core' import { getWorkspaceDB } from '@hcengineering/mongo' import { type StorageAdapter } from '@hcengineering/server-core' @@ -84,7 +84,7 @@ async function processFixJsonMarkupFor ( } if (res !== value) { update[attribute.name] = res - remove.push(getCollaborativeDocId(doc._id, attribute.name)) + remove.push(makeCollaborativeDoc(doc._id, attribute.name)) } } } catch {} diff --git a/models/contact/src/index.ts b/models/contact/src/index.ts index 1c2270b9f72..8ba5cd22525 100644 --- a/models/contact/src/index.ts +++ b/models/contact/src/index.ts @@ -38,11 +38,10 @@ import { IndexKind, type Blob, type Class, + type CollaborativeDoc, type Domain, - type Markup, type Ref, type Timestamp - type CollaborativeDoc } from '@hcengineering/core' import { Collection, diff --git a/packages/core/src/collaboration.ts b/packages/core/src/collaboration.ts index de6d1c6b617..6babecf4266 100644 --- a/packages/core/src/collaboration.ts +++ b/packages/core/src/collaboration.ts @@ -44,14 +44,10 @@ export type CollaborativeDocVersion = string | typeof CollaborativeDocVersionHea export const CollaborativeDocVersionHead = 'HEAD' /** @public */ -export function getCollaborativeDocId (objectId: Ref, objectAttr?: string | undefined): string { - return objectAttr !== undefined && objectAttr !== '' ? `${objectId}%${objectAttr}` : `${objectId}` -} - -/** @public */ -export function getCollaborativeDoc (documentId: string): CollaborativeDoc { +export function makeCollaborativeDoc (objectId: Ref, objectAttr?: string | undefined): CollaborativeDoc { + const storageDocumentId = objectAttr !== undefined && objectAttr !== '' ? `${objectId}%${objectAttr}` : `${objectId}` return collaborativeDocFormat({ - documentId, + documentId: storageDocumentId, versionId: CollaborativeDocVersionHead, lastVersionId: CollaborativeDocVersionHead }) diff --git a/packages/text-editor/src/components/CollaborativeAttributeBox.svelte b/packages/text-editor/src/components/CollaborativeAttributeBox.svelte index 06a2b93c69f..510abbbca89 100644 --- a/packages/text-editor/src/components/CollaborativeAttributeBox.svelte +++ b/packages/text-editor/src/components/CollaborativeAttributeBox.svelte @@ -13,7 +13,7 @@ // limitations under the License. --> @@ -185,7 +199,7 @@ diff --git a/plugins/recruit-resources/src/components/CreateVacancy.svelte b/plugins/recruit-resources/src/components/CreateVacancy.svelte index 71f22734f47..2c6487df54e 100644 --- a/plugins/recruit-resources/src/components/CreateVacancy.svelte +++ b/plugins/recruit-resources/src/components/CreateVacancy.svelte @@ -25,7 +25,8 @@ SortingOrder, fillDefaults, generateId, - getCurrentAccount + getCurrentAccount, + makeCollaborativeDoc } from '@hcengineering/core' import { getEmbeddedLabel } from '@hcengineering/platform' import { Card, InlineAttributeBar, MessageBox, createQuery, getClient } from '@hcengineering/presentation' @@ -40,8 +41,6 @@ EditBox, FocusHandler, IconAttachment, - Label, - Toggle, createFocusManager, showPopup } from '@hcengineering/ui' @@ -88,7 +87,7 @@ attachments: 0, comments: 0, company: '' as Ref, - fullDescription: '', + fullDescription: makeCollaborativeDoc(objectId, 'fullDescription'), location: '', type: typeId as Ref } @@ -103,7 +102,7 @@ $: typeId && templateQ.query(task.class.ProjectType, { _id: typeId }, (result) => { const { _class, _id, description, targetClass, ...templateData } = result[0] - vacancyData = { ...(templateData as unknown as Data), fullDescription: description } + vacancyData = { ...(templateData as unknown as Data) } if (appliedTemplateId !== typeId) { fullDescription = description ?? '' appliedTemplateId = typeId @@ -172,30 +171,42 @@ } const number = (incResult as any).object.sequence + const resId: Ref = generateId() const identifier = `${project?.identifier}-${number}` - const resId = await client.addCollection(tracker.class.Issue, space, parent, tracker.class.Issue, 'subIssues', { - title: template.title + ` (${name})`, - description: template.description, - assignee: template.assignee, - component: template.component, - milestone: template.milestone, - number, - status: project?.defaultIssueStatus as Ref, - priority: template.priority, - rank, - comments: 0, - subIssues: 0, - dueDate: null, - parents: [], - reportedTime: 0, - remainingTime: 0, - estimation: template.estimation, - reports: 0, - relations: [{ _id: id, _class: recruit.class.Vacancy }], - childInfo: [], - kind: taskType._id, - identifier - }) + await client.addCollection( + tracker.class.Issue, + space, + parent, + tracker.class.Issue, + 'subIssues', + { + title: template.title + ` (${name})`, + description: makeCollaborativeDoc(resId, 'description'), + assignee: template.assignee, + component: template.component, + milestone: template.milestone, + number, + status: project?.defaultIssueStatus as Ref, + priority: template.priority, + rank, + comments: 0, + subIssues: 0, + dueDate: null, + parents: [], + reportedTime: 0, + remainingTime: 0, + estimation: template.estimation, + reports: 0, + relations: [{ _id: id, _class: recruit.class.Vacancy }], + childInfo: [], + kind: taskType._id, + identifier, + $markup: { + description: template.description + } + }, + resId + ) if ((template.labels?.length ?? 0) > 0) { const tagElements = await client.findAll(tags.class.TagElement, { _id: { $in: template.labels } }) for (const label of tagElements) { @@ -228,7 +239,7 @@ ...vacancyData, name, description: template?.shortDescription ?? '', - fullDescription, + fullDescription: makeCollaborativeDoc(objectId, 'fullDescription'), private: false, archived: false, number: (incResult as any).object.sequence, @@ -236,7 +247,10 @@ members, autoJoin: typeType.autoJoin ?? false, owners: [getCurrentAccount()._id], - type: typeId + type: typeId, + $markup: { + fullDescription + } }, objectId ) diff --git a/plugins/setting-resources/src/components/WorkspaceSetting.svelte b/plugins/setting-resources/src/components/WorkspaceSetting.svelte index b7802400654..93461684ed6 100644 --- a/plugins/setting-resources/src/components/WorkspaceSetting.svelte +++ b/plugins/setting-resources/src/components/WorkspaceSetting.svelte @@ -19,12 +19,10 @@ import { getClient } from '@hcengineering/presentation' import { WorkspaceSetting } from '@hcengineering/setting' import { FocusHandler, Label, createFocusManager } from '@hcengineering/ui' - import { createEventDispatcher } from 'svelte' import setting from '../plugin' export let visibleNav: boolean = true - const dispatch = createEventDispatcher() let workspaceSettings: WorkspaceSetting | undefined = undefined const client = getClient() @@ -37,7 +35,7 @@ async function onAvatarDone (e: any): Promise { if (workspaceSettings === undefined) { const avatar = await avatarEditor.createAvatar() - await client.createDoc( + await client.createDoc( setting.class.WorkspaceSetting, setting.space.Setting, { icon: avatar.avatar }, diff --git a/plugins/tracker-resources/src/components/CreateIssue.svelte b/plugins/tracker-resources/src/components/CreateIssue.svelte index 0619d28ee88..4865af02694 100644 --- a/plugins/tracker-resources/src/components/CreateIssue.svelte +++ b/plugins/tracker-resources/src/components/CreateIssue.svelte @@ -21,8 +21,8 @@ import core, { Account, Class, + CreateDocData, Doc, - DocData, Ref, SortingOrder, fillDefaults, @@ -42,7 +42,8 @@ MultipleDraftController, SpaceSelector, createQuery, - getClient + getClient, + getMarkup } from '@hcengineering/presentation' import tags, { TagElement, TagReference } from '@hcengineering/tags' import { TaskType, makeRank } from '@hcengineering/task' @@ -190,7 +191,6 @@ if (originalIssue !== undefined && !ignoreOriginal) { const res: IssueDraft = { ...base, - description: originalIssue.description, status: originalIssue.status, priority: originalIssue.priority, component: originalIssue.component, @@ -200,6 +200,9 @@ parentIssue: originalIssue.parents[0]?.parentId, title: `${originalIssue.title} (copy)` } + void getMarkup(originalIssue.description, 'description').then((res) => { + object.description = res + }) void client.findAll(tags.class.TagReference, { attachedTo: originalIssue._id }).then((p) => { object.labels = p }) @@ -446,9 +449,9 @@ const identifier = `${currentProject?.identifier}-${number}` - const value: DocData = { + const value: CreateDocData = { title: getTitle(object.title), - description: object.description, + description: makeCollaborativeDoc(_id, 'description'), assignee: object.assignee, component: object.component, milestone: object.milestone, @@ -478,7 +481,10 @@ relations: relatedTo !== undefined ? [{ _id: relatedTo._id, _class: relatedTo._class }] : [], childInfo: [], kind, - identifier + identifier, + $markup: { + description: object.description + } } await docCreateManager.commit(operations, _id, currentProject, value, 'pre') diff --git a/pods/server/src/__start.ts b/pods/server/src/__start.ts index 663271eeb84..ed475e18adf 100644 --- a/pods/server/src/__start.ts +++ b/pods/server/src/__start.ts @@ -45,6 +45,7 @@ setMetadata(serverNotification.metadata.PushSubject, config.pushSubject) setMetadata(contactPlugin.metadata.LastNameFirst, lastNameFirst) setMetadata(serverCore.metadata.ElasticIndexName, config.elasticIndexName) setMetadata(serverCore.metadata.ElasticIndexVersion, 'v1') +setMetadata(serverCore.metadata.CollaboratorUrl, config.collaboratorUrl) // eslint-disable-next-line @typescript-eslint/no-floating-promises console.log( diff --git a/server-plugins/collaboration-resources/package.json b/server-plugins/collaboration-resources/package.json index 9e0aaa04ed9..23ddd652b4e 100644 --- a/server-plugins/collaboration-resources/package.json +++ b/server-plugins/collaboration-resources/package.json @@ -40,6 +40,8 @@ "@hcengineering/core": "^0.6.32", "@hcengineering/platform": "^0.6.11", "@hcengineering/server-core": "^0.6.1", - "@hcengineering/collaboration": "^0.6.0" + "@hcengineering/server-token": "^0.6.7", + "@hcengineering/collaboration": "^0.6.0", + "@hcengineering/collaborator-client": "^0.6.0" } } diff --git a/server-plugins/collaboration-resources/src/index.ts b/server-plugins/collaboration-resources/src/index.ts index 2999fa69020..087691409ab 100644 --- a/server-plugins/collaboration-resources/src/index.ts +++ b/server-plugins/collaboration-resources/src/index.ts @@ -13,10 +13,100 @@ // limitations under the License. // -import type { CollaborativeDoc, Doc, Tx, TxRemoveDoc } from '@hcengineering/core' -import core, { TxProcessor } from '@hcengineering/core' +import { type CollaboratorClient, getClient as getCollaboratorClient } from '@hcengineering/collaborator-client' +import type { + Class, + CollaborativeDoc, + Data, + Doc, + Hierarchy, + Markup, + MarkupOptions, + MeasureContext, + Ref, + Tx, + TxCUD, + TxCreateDoc, + TxRemoveDoc, + TxUpdateDoc, + WorkspaceId +} from '@hcengineering/core' +import core, { TxProcessor, systemAccountEmail } from '@hcengineering/core' import { removeCollaborativeDoc } from '@hcengineering/collaboration' -import type { TriggerControl } from '@hcengineering/server-core' +import { getMetadata } from '@hcengineering/platform' +import serverCore, { type TriggerControl } from '@hcengineering/server-core' +import { generateToken } from '@hcengineering/server-token' + +function getCollaborator (workspace: WorkspaceId, hierarchy: Hierarchy): CollaboratorClient { + const token = generateToken(systemAccountEmail, workspace, {}) + const collaboratorUrl = getMetadata(serverCore.metadata.CollaboratorUrl) ?? '' + return getCollaboratorClient(hierarchy, workspace, token, collaboratorUrl) +} + +async function updateMarkup ( + ctx: MeasureContext, + _class: Ref>, + doc: Data, + markup: MarkupOptions, + hierarchy: Hierarchy, + workspace: WorkspaceId +): Promise { + if (markup.$markup === undefined) return + + const collaborator = getCollaborator(workspace, hierarchy) + + for (const [k, v] of Object.entries(markup.$markup)) { + const attr = hierarchy.getAttribute(_class, k) + if (!hierarchy.isDerived(attr.type._class, core.class.TypeCollaborativeDoc)) continue + + const collaborativeDoc = (doc as any)[k] as CollaborativeDoc + if (collaborativeDoc !== undefined) { + await ctx.with('update-content', {}, async () => { + await collaborator.updateContent(collaborativeDoc, k, v) + }) + } + } +} + +/** + * @public + */ +export async function MarkupTrigger (tx: TxCUD, control: TriggerControl): Promise { + const etx = TxProcessor.extractTx(tx) as TxCUD + + if (etx._class === core.class.TxCreateDoc) { + return await OnMarkupCreate(tx, control) + } + if (etx._class === core.class.TxUpdateDoc) { + return await OnMarkupUpdate(tx, control) + } + + return [] +} + +async function OnMarkupCreate (tx: Tx, { hierarchy, workspace, ctx }: TriggerControl): Promise { + const createTx = TxProcessor.extractTx(tx) as TxCreateDoc + + if (createTx.attributes.$markup !== undefined) { + await updateMarkup(ctx, createTx.objectClass, createTx.attributes, createTx.attributes, hierarchy, workspace) + } + + return [] +} + +async function OnMarkupUpdate (tx: Tx, { hierarchy, workspace, ctx, findAll }: TriggerControl): Promise { + const updateTx = TxProcessor.extractTx(tx) as TxUpdateDoc + + if (updateTx.operations.$markup !== undefined) { + const rawDoc = (await findAll(updateTx.objectClass, updateTx.objectId))[0] + if (rawDoc !== undefined) { + const doc = TxProcessor.updateDoc2Doc(rawDoc, updateTx) + await updateMarkup(ctx, updateTx.objectClass, doc, updateTx.operations, hierarchy, workspace) + } + } + + return [] +} /** * @public @@ -58,6 +148,7 @@ export async function OnDelete ( // eslint-disable-next-line @typescript-eslint/explicit-function-return-type export default async () => ({ trigger: { + MarkupTrigger, OnDelete } }) diff --git a/server-plugins/collaboration/src/index.ts b/server-plugins/collaboration/src/index.ts index a98c5b3620d..bfd60494768 100644 --- a/server-plugins/collaboration/src/index.ts +++ b/server-plugins/collaboration/src/index.ts @@ -29,6 +29,7 @@ export const serverCollaborationId = 'server-collaboration' as Plugin */ export default plugin(serverCollaborationId, { trigger: { + MarkupTrigger: '' as Resource, OnDelete: '' as Resource } }) diff --git a/server/core/src/plugin.ts b/server/core/src/plugin.ts index 64d60c4868c..086c979eb84 100644 --- a/server/core/src/plugin.ts +++ b/server/core/src/plugin.ts @@ -44,7 +44,8 @@ const serverCore = plugin(serverCoreId, { UploadURL: '' as Metadata, CursorMaxTimeMS: '' as Metadata, ElasticIndexName: '' as Metadata, - ElasticIndexVersion: '' as Metadata + ElasticIndexVersion: '' as Metadata, + CollaboratorUrl: '' as Metadata } }) diff --git a/server/front/src/index.ts b/server/front/src/index.ts index 521bdc878fe..fde340cf8bf 100644 --- a/server/front/src/index.ts +++ b/server/front/src/index.ts @@ -232,7 +232,6 @@ export function start ( gmailUrl: string calendarUrl: string collaboratorUrl: string - collaboratorApiUrl: string brandingUrl?: string previewConfig: string }, @@ -267,7 +266,6 @@ export function start ( GMAIL_URL: config.gmailUrl, CALENDAR_URL: config.calendarUrl, COLLABORATOR_URL: config.collaboratorUrl, - COLLABORATOR_API_URL: config.collaboratorApiUrl, BRANDING_URL: config.brandingUrl, PREVIEW_CONFIG: config.previewConfig, ...(extraConfig ?? {}) From b7b0f73f92f958bc23fc5d5d0f9d86a4a1e48196 Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Wed, 29 May 2024 21:13:14 +0700 Subject: [PATCH 09/34] fix typings Signed-off-by: Alexander Onnikov --- dev/tool/src/clean.ts | 5 +++-- models/server-collaboration/src/index.ts | 2 +- packages/core/src/tx.ts | 2 +- packages/model/src/dsl.ts | 18 +++++++++++------- 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/dev/tool/src/clean.ts b/dev/tool/src/clean.ts index 61189a225ee..7e759810e82 100644 --- a/dev/tool/src/clean.ts +++ b/dev/tool/src/clean.ts @@ -32,6 +32,7 @@ import core, { type Doc, type Domain, type MeasureContext, + type Mixin, type Ref, type Status, type StatusCategory, @@ -922,7 +923,7 @@ export async function restoreHrTaskTypesFromUpdates ( const taskTargetClassId = `${taskTypeId}:type:mixin` as Ref> const ofClassClass = hierarchy.getClass(recruit.class.Applicant) - await db.collection>(DOMAIN_TX).insertOne({ + await db.collection>>(DOMAIN_TX).insertOne({ _id: generateId(), _class: core.class.TxCreateDoc, space: core.space.Tx, @@ -975,7 +976,7 @@ export async function restoreHrTaskTypesFromUpdates ( const targetClassId = `${projectTypeId}:type:mixin` as Ref> const ofClassClass = hierarchy.getClass(recruit.class.Vacancy) - await db.collection>(DOMAIN_TX).insertOne({ + await db.collection>>(DOMAIN_TX).insertOne({ _id: generateId(), _class: core.class.TxCreateDoc, space: core.space.Tx, diff --git a/models/server-collaboration/src/index.ts b/models/server-collaboration/src/index.ts index b64da7b9cce..74e512a694c 100644 --- a/models/server-collaboration/src/index.ts +++ b/models/server-collaboration/src/index.ts @@ -23,7 +23,7 @@ export { serverCollaborationId } from '@hcengineering/server-collaboration' export function createModel (builder: Builder): void { builder.createDoc(serverCore.class.Trigger, core.space.Model, { - trigger: serverCollaboration.trigger.OnCreate + trigger: serverCollaboration.trigger.OnDelete }) builder.createDoc(serverCore.class.Trigger, core.space.Model, { trigger: serverCollaboration.trigger.MarkupTrigger diff --git a/packages/core/src/tx.ts b/packages/core/src/tx.ts index 889211283c8..4cd00c11ba8 100644 --- a/packages/core/src/tx.ts +++ b/packages/core/src/tx.ts @@ -320,7 +320,7 @@ export interface SpaceUpdate { /** * @public */ -export interface MarkupOptions { +export interface MarkupOptions> { $markup?: Partial>> } diff --git a/packages/model/src/dsl.ts b/packages/model/src/dsl.ts index b68cdcf3381..8acf00a67a5 100644 --- a/packages/model/src/dsl.ts +++ b/packages/model/src/dsl.ts @@ -265,21 +265,25 @@ function _generateTx (tx: ClassTxes): Tx[] { [ClassifierKind.INTERFACE]: core.class.Interface, [ClassifierKind.MIXIN]: core.class.Mixin } - const createTx = txFactory.createTxCreateDoc( + const createTx = txFactory.createTxCreateDoc( _cl[tx.kind], core.space.Model, { ...(tx.domain !== undefined ? { domain: tx.domain } : {}), kind: tx.kind, + label: tx.label, + icon: tx.icon, ...(tx.kind === ClassifierKind.INTERFACE ? { extends: tx.implements } : { extends: tx.extends, implements: tx.implements }), - label: tx.label, - icon: tx.icon, - shortLabel: tx.shortLabel, - sortingKey: tx.sortingKey, - filteringKey: tx.filteringKey, - pluralLabel: tx.pluralLabel + ...(tx.kind === ClassifierKind.INTERFACE + ? { extends: tx.implements } + : { + shortLabel: tx.shortLabel, + sortingKey: tx.sortingKey, + filteringKey: tx.filteringKey, + pluralLabel: tx.pluralLabel + }) }, objectId ) From c19f8048e2c5446ca36c0fcdc86c395cfc2113cb Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Thu, 30 May 2024 12:42:36 +0700 Subject: [PATCH 10/34] Migrate collaborative markup to storage Signed-off-by: Alexander Onnikov --- models/core/package.json | 4 +- models/core/src/migration.ts | 94 ++++++++++++++++++- packages/core/src/tx.ts | 6 +- packages/text/src/ydoc.ts | 29 +++++- .../src/utils/collaborative-doc.ts | 2 - 5 files changed, 123 insertions(+), 12 deletions(-) diff --git a/models/core/package.json b/models/core/package.json index fee665da8bc..cf515d34a5b 100644 --- a/models/core/package.json +++ b/models/core/package.json @@ -31,6 +31,8 @@ "@hcengineering/core": "^0.6.32", "@hcengineering/model": "^0.6.11", "@hcengineering/platform": "^0.6.11", - "@hcengineering/storage": "^0.6.0" + "@hcengineering/storage": "^0.6.0", + "@hcengineering/collaboration": "^0.6.0", + "@hcengineering/text": "^0.6.5" } } diff --git a/models/core/src/migration.ts b/models/core/src/migration.ts index 4536dccd766..f48b6a57dd7 100644 --- a/models/core/src/migration.ts +++ b/models/core/src/migration.ts @@ -13,6 +13,7 @@ // limitations under the License. // +import { saveCollaborativeDoc } from '@hcengineering/collaboration' import core, { DOMAIN_BLOB, DOMAIN_DOC_INDEX_STATE, @@ -22,7 +23,12 @@ import core, { coreId, generateId, isClassIndexable, + makeCollaborativeDoc, + type AnyAttribute, type Blob, + type Doc, + type Domain, + type MeasureContext, type Ref, type Space, type Status, @@ -33,10 +39,14 @@ import { tryMigrate, tryUpgrade, type MigrateOperation, + type MigrateUpdate, type MigrationClient, + type MigrationDocumentQuery, + type MigrationIterator, type MigrationUpgradeClient } from '@hcengineering/model' -import { type StorageAdapterEx } from '@hcengineering/storage' +import { type StorageAdapter, type StorageAdapterEx } from '@hcengineering/storage' +import { markupToYDoc } from '@hcengineering/text' import { DOMAIN_SPACE } from './security' async function migrateStatusesToModel (client: MigrationClient): Promise { @@ -143,6 +153,83 @@ async function migrateStatusTransactions (client: MigrationClient): Promise { + const ctx = new MeasureMetricsContext('migrate_content', {}) + const storageAdapter = client.storageAdapter as StorageAdapter + + const hierarchy = client.hierarchy + const classes = hierarchy.getDescendants(core.class.Doc) + for (const _class of classes) { + const domain = hierarchy.findDomain(_class) + if (domain === undefined) continue + + const attributes = hierarchy.getAllAttributes(_class) + const filtered = Array.from(attributes.values()).filter((attribute) => { + return hierarchy.isDerived(attribute.type._class, core.class.TypeCollaborativeDoc) + }) + if (filtered.length === 0) continue + + const iterator = await client.traverse(domain, { _class }) + try { + console.log('processing', _class) + await processMigrateContentFor(ctx, domain, filtered, client, storageAdapter, iterator) + } finally { + await iterator.close() + } + } +} + +async function processMigrateContentFor ( + ctx: MeasureContext, + domain: Domain, + attributes: AnyAttribute[], + client: MigrationClient, + storageAdapter: StorageAdapter, + iterator: MigrationIterator +): Promise { + let processed = 0 + while (true) { + const docs = await iterator.next(1000) + if (docs === null || docs.length === 0) { + break + } + + const operations: { filter: MigrationDocumentQuery, update: MigrateUpdate }[] = [] + + for (const doc of docs) { + const update: MigrateUpdate = {} + + for (const attribute of attributes) { + const value = (doc as any)[attribute.name] as string + if (value != null && value.startsWith('{')) { + const collaborativeDoc = makeCollaborativeDoc(doc._id, attribute.name) + update[attribute.name] = collaborativeDoc + + const ydoc = markupToYDoc(value, attribute.name) + await saveCollaborativeDoc( + storageAdapter, + client.workspaceId, + collaborativeDoc, + ydoc, + ctx + ) + } + } + + if (Object.keys(update).length > 0) { + operations.push({ filter: { _id: doc._id }, update }) + } + } + + if (operations.length > 0) { + await client.bulk(domain, operations) + } + + processed += docs.length + console.log('...processed', processed) + } +} + export const coreOperation: MigrateOperation = { async migrate (client: MigrationClient): Promise { // We need to delete all documents in doc index state for missing classes @@ -180,8 +267,13 @@ export const coreOperation: MigrateOperation = { } }, { +<<<<<<< HEAD state: 'old-statuses-transactions', func: migrateStatusTransactions +======= + state: 'collaborative-content-to-storage', + func: migrateCollaborativeContentToStorage +>>>>>>> 8843cd60a (Migrate collaborative markup to storage) } ]) }, diff --git a/packages/core/src/tx.ts b/packages/core/src/tx.ts index 4cd00c11ba8..41359f390b8 100644 --- a/packages/core/src/tx.ts +++ b/packages/core/src/tx.ts @@ -197,7 +197,7 @@ export type ArrayAsElement = { /** * @public */ -export type ColalborativeDocAsMarkup = { +export type CollaborativeDocAsMarkup = { [P in keyof T]-?: T[P] extends CollaborativeDoc ? Markup : never } @@ -321,7 +321,7 @@ export interface SpaceUpdate { * @public */ export interface MarkupOptions> { - $markup?: Partial>> + $markup?: Partial>> } /** @@ -392,7 +392,7 @@ export abstract class TxProcessor implements WithTx { } static createDoc2Doc(tx: TxCreateDoc, doClone = true): T { - const attributes = Object.fromEntries(Object.entries(tx.attributes).filter(([k]) => k.startsWith('$'))) + const attributes = Object.fromEntries(Object.entries(tx.attributes).filter(([key]) => !key.startsWith('$'))) // eslint-disable-next-line @typescript-eslint/consistent-type-assertions return { ...(doClone ? clone(attributes) : attributes), diff --git a/packages/text/src/ydoc.ts b/packages/text/src/ydoc.ts index 5ae1384f915..ba44624a107 100644 --- a/packages/text/src/ydoc.ts +++ b/packages/text/src/ydoc.ts @@ -13,11 +13,30 @@ // limitations under the License. // +import { Markup } from '@hcengineering/core' import { Extensions, getSchema } from '@tiptap/core' import { Node, Schema } from 'prosemirror-model' -import { yDocToProsemirrorJSON } from 'y-prosemirror' -import { Doc, applyUpdate } from 'yjs' +import { yDocToProsemirrorJSON, prosemirrorToYDoc } from 'y-prosemirror' +import { Doc as YDoc, applyUpdate } from 'yjs' import { defaultExtensions } from './extensions' +import { MarkupNode } from './markup/model' +import { jsonToMarkup, markupToPmNode } from './markup/utils' + +/** + * @public + */ +export function markupToYDoc (markup: Markup, field: string): YDoc { + const node = markupToPmNode(markup) + return prosemirrorToYDoc(node, field) +} + +/** + * @public + */ +export function yDocToMarkup (ydoc: YDoc, field: string): Markup { + const json = yDocToProsemirrorJSON(ydoc, field) + return jsonToMarkup(json as MarkupNode) +} /** * Get ProseMirror node from Y.Doc content @@ -30,7 +49,7 @@ export function yDocContentToNode ( schema?: Schema, extensions?: Extensions ): Node { - const ydoc = new Doc() + const ydoc = new YDoc() const uint8arr = new Uint8Array(content) applyUpdate(ydoc, uint8arr) @@ -42,7 +61,7 @@ export function yDocContentToNode ( * * @public */ -export function yDocToNode (ydoc: Doc, field?: string, schema?: Schema, extensions?: Extensions): Node { +export function yDocToNode (ydoc: YDoc, field?: string, schema?: Schema, extensions?: Extensions): Node { schema ??= getSchema(extensions ?? defaultExtensions) try { @@ -65,7 +84,7 @@ export function yDocContentToNodes (content: ArrayBuffer, schema?: Schema, exten const nodes: Node[] = [] try { - const ydoc = new Doc() + const ydoc = new YDoc() const uint8arr = new Uint8Array(content) applyUpdate(ydoc, uint8arr) diff --git a/server/collaboration/src/utils/collaborative-doc.ts b/server/collaboration/src/utils/collaborative-doc.ts index debe69b8f8e..2ba7f346f7f 100644 --- a/server/collaboration/src/utils/collaborative-doc.ts +++ b/server/collaboration/src/utils/collaborative-doc.ts @@ -83,8 +83,6 @@ export async function loadCollaborativeDoc ( return await ctx.with('loadCollaborativeDoc', { type: 'content' }, async (ctx) => { for (const source of sources) { const { documentId, versionId } = collaborativeDocParse(source) - - ctx.info('loading collaborative document', { source }) const ydoc = await loadCollaborativeDocVersion(ctx, storageAdapter, workspace, documentId, versionId) if (ydoc !== undefined) { From f23a55a41b79c2947080f7822bc71ad205aaaa52 Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Fri, 31 May 2024 15:03:56 +0700 Subject: [PATCH 11/34] fix issue preview description Signed-off-by: Alexander Onnikov --- .../src/components/CollaborativeAttributeBox.svelte | 2 +- .../src/components/issues/IssuePreview.svelte | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/packages/text-editor/src/components/CollaborativeAttributeBox.svelte b/packages/text-editor/src/components/CollaborativeAttributeBox.svelte index 34394293596..9c3dd3034b0 100644 --- a/packages/text-editor/src/components/CollaborativeAttributeBox.svelte +++ b/packages/text-editor/src/components/CollaborativeAttributeBox.svelte @@ -13,7 +13,7 @@ // limitations under the License. -->