Skip to content

UBER-1263 Fix blobs migration on new workspaces #6586

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Sep 16, 2024
Merged

UBER-1263 Fix blobs migration on new workspaces #6586

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
af97d8b
Remove blobs migration
aonnikov Sep 16, 2024
8cd9fa5
Upgrade created workspace
aonnikov Sep 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 1 addition & 51 deletions models/core/src/migration.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@

import { saveCollaborativeDoc } from '@hcengineering/collaboration'
import core, {
DOMAIN_BLOB,
DOMAIN_DOC_INDEX_STATE,
DOMAIN_SPACE,
DOMAIN_STATUS,
Expand All @@ -28,11 +27,9 @@ import core, {
isClassIndexable,
makeCollaborativeDoc,
type AnyAttribute,
type Blob,
type Doc,
type Domain,
type MeasureContext,
type Ref,
type Space,
type Status,
type TxCreateDoc
Expand All @@ -48,7 +45,7 @@ import {
type MigrationIterator,
type MigrationUpgradeClient
} from '@hcengineering/model'
import { type StorageAdapter, type StorageAdapterEx } from '@hcengineering/storage'
import { type StorageAdapter } from '@hcengineering/storage'
import { markupToYDoc } from '@hcengineering/text'

async function migrateStatusesToModel (client: MigrationClient): Promise<void> {
Expand Down Expand Up @@ -278,7 +275,6 @@ export const coreOperation: MigrateOperation = {
}
}
)
const exAdapter: StorageAdapterEx = client.storageAdapter as StorageAdapterEx
await tryMigrate(client, coreId, [
{
state: 'statuses-to-model',
Expand All @@ -292,12 +288,6 @@ export const coreOperation: MigrateOperation = {
state: 'add-spaces-owner',
func: migrateSpacesOwner
},
{
state: 'storage_blobs_v1',
func: async (client: MigrationClient) => {
await migrateBlobData(exAdapter, client)
}
},
{
state: 'old-statuses-transactions',
func: migrateStatusTransactions
Expand Down Expand Up @@ -340,43 +330,3 @@ export const coreOperation: MigrateOperation = {
])
}
}
async function migrateBlobData (exAdapter: StorageAdapterEx, client: MigrationClient): Promise<void> {
const ctx = new MeasureMetricsContext('storage_upgrade', {})

for (const [provider, adapter] of exAdapter.adapters?.entries() ?? []) {
if (!(await adapter.exists(ctx, client.workspaceId))) {
continue
}
const blobs = await adapter.listStream(ctx, client.workspaceId)
const bulk = new Map<Ref<Blob>, Blob>()
try {
const push = async (force: boolean): Promise<void> => {
if (bulk.size > 1000 || force) {
await client.deleteMany(DOMAIN_BLOB, { _id: { $in: Array.from(bulk.keys()) } })
await client.create(DOMAIN_BLOB, Array.from(bulk.values()))
bulk.clear()
}
}
while (true) {
const blob = await blobs.next()
if (blob === undefined) {
break
}
// We need to state details for blob.
const blobData = await adapter.stat(ctx, client.workspaceId, blob._id)
if (blobData !== undefined) {
bulk.set(blobData._id, {
...blobData,
provider
})
}
await push(false)
}
await push(true)
} catch (err: any) {
ctx.error('Error during blob migration', { error: err.message })
} finally {
await blobs.close()
}
}
}
17 changes: 16 additions & 1 deletion server/workspace-service/src/ws-operations.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -200,9 +200,24 @@ export async function createWorkspace (
ctx.info('Starting init script if any')
await initializeWorkspace(ctx, branding, wsUrl, storageAdapter, client, ctxModellogger, async (value) => {
ctx.info('Init script progress', { value })
await handleWsEvent?.('progress', version, 30 + Math.round((Math.min(value, 100) / 100) * 70))
await handleWsEvent?.('progress', version, 30 + Math.round((Math.min(value, 100) / 100) * 60))
})

await upgradeWorkspace(
ctx,
version,
txes,
migrationOperation,
workspaceInfo,
ctxModellogger,
async (event, version, value) => {
ctx.info('Init script progress', { event, value })
await handleWsEvent?.('progress', version, 90 + Math.round((Math.min(value, 100) / 100) * 10))
},
false,
false
)

await pipeline.close()

await handleWsEvent?.('create-done', version, 100, '')
Expand Down