UBERF-9383: Fix workspace init and unified import

.vscode/launch.json

@@ -138,11 +138,10 @@
       "env": {
         "MONGO_URL": "mongodb://localhost:27017",
         "DB_URL": "mongodb://localhost:27017",
-        // "DB_URL": "postgresql://postgres:example@localhost:5432",
         // "DB_URL": "postgresql://root@host.docker.internal:26257/defaultdb?sslmode=disable",
         "SERVER_SECRET": "secret",
-        "REGION_INFO":"|Mongo;pg|Postgres;cockroach|CockroachDB",
-        "TRANSACTOR_URL": "ws://host.docker.internal:3333,ws://host.docker.internal:3331;;pg,ws://host.docker.internal:3332;;cockroach",
+        "REGION_INFO":"|Mongo;cockroach|CockroachDB",
+        "TRANSACTOR_URL": "ws://host.docker.internal:3333,ws://host.docker.internal:3332;;cockroach",
         "ACCOUNTS_URL": "http://localhost:3000",
         "ACCOUNT_PORT": "3000",
         "FRONT_URL": "http://localhost:8080",
@@ -154,8 +153,6 @@
         "MINIO_SECRET_KEY": "minioadmin",
         "MINIO_ENDPOINT": "localhost"
         // "DISABLE_SIGNUP": "true",
-        // "INIT_SCRIPT_URL": "https://raw.githubusercontent.com/hcengineering/init/main/script.yaml",
-        // "INIT_WORKSPACE": "onboarding",
       },
       "runtimeVersion": "20",
       "runtimeArgs": ["--nolazy", "-r", "ts-node/register"],
@@ -187,8 +184,6 @@
         "MINIO_SECRET_KEY": "minioadmin",
         "MINIO_ENDPOINT": "localhost"
         // "DISABLE_SIGNUP": "true",
-        // "INIT_SCRIPT_URL": "https://raw.githubusercontent.com/hcengineering/init/main/script.yaml",
-        // "INIT_WORKSPACE": "onboarding",
       },
       "runtimeVersion": "20",
       "runtimeArgs": ["--nolazy", "-r", "ts-node/register"],
@@ -236,8 +231,8 @@
         "WS_OPERATION": "all+backup",
         "BACKUP_STORAGE": "minio|minio?accessKey=minioadmin&secretKey=minioadmin",
         "BACKUP_BUCKET": "dev-backups",
-        // "INIT_SCRIPT_URL": "https://raw.githubusercontent.com/hcengineering/init/main/script.yaml",
-        // "INIT_WORKSPACE": "onboarding",
+        // "INIT_REPO_DIR": "${workspaceRoot}/pods/workspace/init",
+        // "INIT_WORKSPACE": "staging-dev"
       },
       "runtimeVersion": "20",
       "runtimeArgs": ["--nolazy", "-r", "ts-node/register"],
@@ -269,8 +264,8 @@
         "WS_OPERATION": "all+backup",
         "BACKUP_STORAGE": "minio|minio?accessKey=minioadmin&secretKey=minioadmin",
         "BACKUP_BUCKET": "dev-backups",
-        // "INIT_SCRIPT_URL": "https://raw.githubusercontent.com/hcengineering/init/main/script.yaml",
-        // "INIT_WORKSPACE": "onboarding",
+        // "INIT_REPO_DIR": "${workspaceRoot}/pods/workspace/init",
+        // "INIT_WORKSPACE": "staging-dev"
       },
       "runtimeVersion": "20",
       "runtimeArgs": ["--nolazy", "-r", "ts-node/register"],

dev/docker-compose.yaml

@@ -80,7 +80,7 @@ services:
       # Pass only one region to disallow selection for new workspaces.Ø
       - REGION_INFO=|Mongo;cockroach|CockroachDB
       # - REGION_INFO=cockroach|CockroachDB
-      - TRANSACTOR_URL=ws://host.docker.internal:3333,ws://host.docker.internal:3331;;pg,ws://host.docker.internal:3332;;cockroach,
+      - TRANSACTOR_URL=ws://host.docker.internal:3333,ws://host.docker.internal:3332;;cockroach,
       - SES_URL=
       - STORAGE_CONFIG=${STORAGE_CONFIG}
       - FRONT_URL=http://host.docker.internal:8087
@@ -91,8 +91,6 @@ services:
       - ACCOUNTS_URL=http://host.docker.internal:3000
       - BRANDING_PATH=/var/cfg/branding.json
       # - DISABLE_SIGNUP=true
-      # - INIT_SCRIPT_URL=https://raw.githubusercontent.com/hcengineering/init/main/script.yaml
-      # - INIT_WORKSPACE=onboarding
     restart: unless-stopped
   stats:
     image: hardcoreeng/stats
@@ -126,9 +124,9 @@ services:
       - ACCOUNTS_URL=http://host.docker.internal:3000
       - BRANDING_PATH=/var/cfg/branding.json
       # - PARALLEL=2
-      - INIT_WORKSPACE=test
       - BACKUP_STORAGE=${BACKUP_STORAGE_CONFIG}
       - BACKUP_BUCKET=${BACKUP_BUCKET_NAME}
+      # - INIT_WORKSPACE=staging-dev
     restart: unless-stopped
   workspace_cockroach:
     image: hardcoreeng/workspace
@@ -153,9 +151,9 @@ services:
       - ACCOUNTS_URL=http://host.docker.internal:3000
       - BRANDING_PATH=/var/cfg/branding.json
       # - PARALLEL=2
-      # - INIT_WORKSPACE=onboarding
       - BACKUP_STORAGE=${BACKUP_STORAGE_CONFIG}
       - BACKUP_BUCKET=${BACKUP_BUCKET_NAME}
+      # - INIT_WORKSPACE=staging-dev
     restart: unless-stopped
   collaborator:
     image: hardcoreeng/collaborator

dev/tool/src/db.ts

@@ -334,7 +334,7 @@ export async function moveAccountDbFromMongoToPG (
       }
 
       if (workspacesCount % 100 === 0) {
-        ctx.info(`Migrated ${workspacesCount} invites...`)
+        ctx.info(`Migrated ${workspacesCount} workspaces...`)
       }
     }
     ctx.info(`Migrated ${workspacesCount} workspaces with ${membersCount} member assignments`)

dev/tool/src/index.ts

@@ -46,6 +46,7 @@ import {
   AccountRole,
   MeasureMetricsContext,
   metricsToString,
+  type PersonId,
   type Data,
   type Tx,
   type Version,
@@ -304,7 +305,7 @@ export function devTool (
         const measureCtx = new MeasureMetricsContext('create-workspace', {})
         const brandingObj =
           cmd.branding !== undefined || cmd.init !== undefined ? { key: cmd.branding, initWorkspace: cmd.init } : null
-        const socialId = await db.socialId.findOne({ key: socialString })
+        const socialId = await db.socialId.findOne({ key: socialString as PersonId })
         if (socialId == null) {
           throw new Error(`Social id ${socialString} not found`)
         }

packages/account-client/src/client.ts

@@ -17,7 +17,8 @@ import {
   BackupStatus,
   Data,
   type Person,
-  PersonUuid,
+  type PersonUuid,
+  type PersonInfo,
   SocialId,
   Version,
   type WorkspaceInfoWithStatus,
@@ -73,6 +74,7 @@ export interface AccountClient {
   signUp: (email: string, password: string, first: string, last: string) => Promise<LoginInfo>
   login: (email: string, password: string) => Promise<LoginInfo>
   getPerson: () => Promise<Person>
+  getPersonInfo: (account: PersonUuid) => Promise<PersonInfo>
   getSocialIds: () => Promise<SocialId[]>
   getWorkspaceMembers: () => Promise<WorkspaceMemberInfo[]>
   updateWorkspaceRole: (account: string, role: AccountRole) => Promise<void>
@@ -399,6 +401,15 @@ class AccountClientImpl implements AccountClient {
     return await this.rpc(request)
   }
 
+  async getPersonInfo (account: PersonUuid): Promise<PersonInfo> {
+    const request = {
+      method: 'getPersonInfo' as const,
+      params: [account]
+    }
+
+    return await this.rpc(request)
+  }
+
   async getSocialIds (): Promise<SocialId[]> {
     const request = {
       method: 'getSocialIds' as const,

packages/core/src/classes.ts

@@ -538,6 +538,10 @@ export interface Person {
   city?: string
 }
 
+export interface PersonInfo extends BasePerson {
+  socialIds: PersonId[]
+}
+
 /**
  * @public
  */

packages/importer/src/huly/unified.ts

@@ -16,11 +16,13 @@
 import { type Attachment } from '@hcengineering/attachment'
 import contact, { Employee, type Person } from '@hcengineering/contact'
 import {
+  buildSocialIdString,
   type Class,
   type Doc,
   generateId,
   PersonId,
   type Ref,
+  SocialIdType,
   type Space,
   type TxOperations
 } from '@hcengineering/core'
@@ -330,24 +332,26 @@ interface AttachmentMetadata {
 }
 
 export class UnifiedFormatImporter {
+  private readonly importerEmailPlaceholder = 'newuser@huly.io'
+  private readonly importerNamePlaceholder = 'New User'
   private readonly pathById = new Map<Ref<Doc>, string>()
   private readonly refMetaByPath = new Map<string, ReferenceMetadata>()
   private readonly fileMetaByPath = new Map<string, AttachmentMetadata>()
   private readonly ctrlDocTemplateIdByPath = new Map<string, Ref<ControlledDocument>>()
 
   private personsByName = new Map<string, Ref<Person>>()
-  // private accountsByEmail = new Map<string, Ref<PersonAccount>>()
-  // private employeesByName = new Map<string, Ref<Employee>>()
+  private employeesByName = new Map<string, Ref<Employee>>()
 
   constructor (
     private readonly client: TxOperations,
     private readonly fileUploader: FileUploader,
-    private readonly logger: Logger
+    private readonly logger: Logger,
+    private readonly importerSocialId?: PersonId,
+    private readonly importerPerson?: Ref<Person>
   ) {}
 
   private async initCaches (): Promise<void> {
     await this.cachePersonsByNames()
-    await this.cacheAccountsByEmails()
     await this.cacheEmployeesByName()
   }
 
@@ -588,31 +592,31 @@ export class UnifiedFormatImporter {
     if (name === undefined) {
       return undefined
     }
+
+    if (name === this.importerNamePlaceholder && this.importerPerson != null) {
+      return this.importerPerson
+    }
     const person = this.personsByName.get(name)
     if (person === undefined) {
       throw new Error(`Person not found: ${name}`)
     }
     return person
   }
 
-  private findAccountByEmail (email: string): PersonId {
-    // TODO: FIXME
-    throw new Error('Not implemented')
-    // const account = this.accountsByEmail.get(email)
-    // if (account === undefined) {
-    //   throw new Error(`Account not found: ${email}`)
-    // }
-    // return account
+  private getSocialIdByEmail (email: string): PersonId {
+    if (email === this.importerEmailPlaceholder && this.importerSocialId != null) {
+      return this.importerSocialId
+    }
+
+    return buildSocialIdString({ type: SocialIdType.EMAIL, value: email })
   }
 
   private findEmployeeByName (name: string): Ref<Employee> {
-    // TODO: FIXME
-    throw new Error('Not implemented')
-    // const employee = this.employeesByName.get(name)
-    // if (employee === undefined) {
-    //   throw new Error(`Employee not found: ${name}`)
-    // }
-    // return employee
+    const employee = this.employeesByName.get(name)
+    if (employee === undefined) {
+      throw new Error(`Employee not found: ${name}`)
+    }
+    return employee
   }
 
   private async processDocumentsRecursively (
@@ -752,7 +756,7 @@ export class UnifiedFormatImporter {
         }
         return {
           text: comment.text,
-          author: this.findAccountByEmail(comment.author),
+          author: this.getSocialIdByEmail(comment.author),
           attachments
         }
       })
@@ -789,9 +793,9 @@ export class UnifiedFormatImporter {
       defaultIssueStatus:
         projectHeader.defaultIssueStatus !== undefined ? { name: projectHeader.defaultIssueStatus } : undefined,
       owners:
-        projectHeader.owners !== undefined ? projectHeader.owners.map((email) => this.findAccountByEmail(email)) : [],
+        projectHeader.owners !== undefined ? projectHeader.owners.map((email) => this.getSocialIdByEmail(email)) : [],
       members:
-        projectHeader.members !== undefined ? projectHeader.members.map((email) => this.findAccountByEmail(email)) : [],
+        projectHeader.members !== undefined ? projectHeader.members.map((email) => this.getSocialIdByEmail(email)) : [],
       docs: []
     }
   }
@@ -805,9 +809,9 @@ export class UnifiedFormatImporter {
       archived: spaceHeader.archived ?? false,
       description: spaceHeader.description,
       emoji: spaceHeader.emoji,
-      owners: spaceHeader.owners !== undefined ? spaceHeader.owners.map((email) => this.findAccountByEmail(email)) : [],
+      owners: spaceHeader.owners !== undefined ? spaceHeader.owners.map((email) => this.getSocialIdByEmail(email)) : [],
       members:
-        spaceHeader.members !== undefined ? spaceHeader.members.map((email) => this.findAccountByEmail(email)) : [],
+        spaceHeader.members !== undefined ? spaceHeader.members.map((email) => this.getSocialIdByEmail(email)) : [],
       docs: []
     }
   }
@@ -819,11 +823,11 @@ export class UnifiedFormatImporter {
       private: spaceHeader.private ?? false,
       archived: spaceHeader.archived ?? false,
       description: spaceHeader.description,
-      owners: spaceHeader.owners?.map((email) => this.findAccountByEmail(email)) ?? [],
-      members: spaceHeader.members?.map((email) => this.findAccountByEmail(email)) ?? [],
-      qualified: spaceHeader.qualified !== undefined ? this.findAccountByEmail(spaceHeader.qualified) : undefined,
-      manager: spaceHeader.manager !== undefined ? this.findAccountByEmail(spaceHeader.manager) : undefined,
-      qara: spaceHeader.qara !== undefined ? this.findAccountByEmail(spaceHeader.qara) : undefined,
+      owners: spaceHeader.owners?.map((email) => this.getSocialIdByEmail(email)) ?? [],
+      members: spaceHeader.members?.map((email) => this.getSocialIdByEmail(email)) ?? [],
+      qualified: spaceHeader.qualified !== undefined ? this.getSocialIdByEmail(spaceHeader.qualified) : undefined,
+      manager: spaceHeader.manager !== undefined ? this.getSocialIdByEmail(spaceHeader.manager) : undefined,
+      qara: spaceHeader.qara !== undefined ? this.getSocialIdByEmail(spaceHeader.qara) : undefined,
       docs: []
     }
   }
@@ -950,30 +954,18 @@ export class UnifiedFormatImporter {
       }, new Map())
   }
 
-  private async cacheAccountsByEmails (): Promise<void> {
-    // TODO: FIXME
-    throw new Error('Not implemented')
-    // const accounts = await this.client.findAll(contact.class.PersonAccount, {})
-    // this.accountsByEmail = accounts.reduce((map, account) => {
-    //   map.set(account.email, account._id)
-    //   return map
-    // }, new Map())
-  }
-
   private async cacheEmployeesByName (): Promise<void> {
-    // TODO: FIXME
-    throw new Error('Not implemented')
-    // this.employeesByName = (await this.client.findAll(contact.mixin.Employee, {}))
-    //   .map((employee) => {
-    //     return {
-    //       _id: employee._id,
-    //       name: employee.name.split(',').reverse().join(' ')
-    //     }
-    //   })
-    //   .reduce((refByName, employee) => {
-    //     refByName.set(employee.name, employee._id)
-    //     return refByName
-    //   }, new Map())
+    this.employeesByName = (await this.client.findAll(contact.mixin.Employee, {}))
+      .map((employee) => {
+        return {
+          _id: employee._id,
+          name: employee.name.split(',').reverse().join(' ')
+        }
+      })
+      .reduce((refByName, employee) => {
+        refByName.set(employee.name, employee._id)
+        return refByName
+      }, new Map())
   }
 
   private async collectFileMetadata (folderPath: string): Promise<void> {

pods/workspace/.gitignore

@@ -0,0 +1 @@
+init