diff --git a/src/lib/migrate.ts b/src/lib/migrate.ts index 94e8aa91d7..88932b6183 100644 --- a/src/lib/migrate.ts +++ b/src/lib/migrate.ts @@ -1,4 +1,4 @@ -import { NoteStorage, AttachmentData } from './db/types' +import { NoteStorage, AttachmentData, NoteDoc } from './db/types' import { SerializedWorkspace } from '../cloud/interfaces/db/workspace' import { uploadFile, buildTeamFileUrl } from '../cloud/api/teams/files' import { createDocREST } from '../cloud/api/rest/doc' @@ -94,56 +94,120 @@ async function* createMigrationIter( const jobCount = attachments.length * 2 + notes.length let jobsCompleted = 0 - for (const [id, attachment] of attachments) { + const attachmentsQueue = attachments.map(([id, attachment]) => ({ + id, + attachment, + retries: 0, + })) + while (attachmentsQueue.length > 0) { + const job = attachmentsQueue.shift() + if (job == null) break + const { attachment, id, retries } = job + yield { jobCount, jobsCompleted, stage: { name: 'attachments', handling: id, subStage: 'setup' }, } - const data = await attachment.getData() - const file = await loadFile(data, id) - yield { - jobCount, - jobsCompleted: ++jobsCompleted, - stage: { name: 'attachments', handling: id, subStage: 'upload' }, + + try { + const data = await attachment.getData() + const file = await loadFile(data, id) + yield { + jobCount, + jobsCompleted: ++jobsCompleted, + stage: { name: 'attachments', handling: id, subStage: 'upload' }, + } + const upload = await uploadFile(workspace.teamId, file) + attachmentSources.push([ + id, + buildTeamFileUrl(workspace.teamId, upload.file.name), + ]) + } catch (err) { + if ( + retries > 9 || + (err.response instanceof Response && !isRetryable(err.response)) + ) { + throw err + } else { + attachmentsQueue.push({ attachment, id, retries: retries + 1 }) + } } - const upload = await uploadFile(workspace.teamId, file) - attachmentSources.push([ - id, - buildTeamFileUrl(workspace.teamId, upload.file.name), - ]) } - for (const [, note] of notes) { + const notesQueue = notes.map(([, note]) => ({ + note, + retries: 0, + })) + + while (notesQueue.length > 0) { + const job = notesQueue.shift() + if (job == null) break + const { note, retries } = job + yield { jobCount, - jobsCompleted: ++jobsCompleted, + jobsCompleted, stage: { name: 'document', - handling: `${note.folderPathname}/${note.title}`, + handling: getNotePath(note), }, } - const content = replaceAttachments(note.content, attachmentSources) - await createDocREST({ - workspaceId: workspace.id, - teamId: workspace.teamId, - content, - title: note.title, - tags: note.tags, - path: note.folderPathname, - generated: true, - events: true, - }) + + try { + const content = replaceAttachments(note.content, attachmentSources) + await createDocREST({ + workspaceId: workspace.id, + teamId: workspace.teamId, + content, + title: note.title, + tags: note.tags, + path: note.folderPathname, + generated: true, + events: true, + }) + + yield { + jobCount, + jobsCompleted: ++jobsCompleted, + stage: { + name: 'document', + handling: getNotePath(note), + }, + } + } catch (err) { + if ( + retries > 9 || + (err.response instanceof Response && !isRetryable(err.response)) + ) { + throw err + } else { + notesQueue.push({ note, retries: retries + 1 }) + } + } } + return { jobCount, jobsCompleted, stage: { name: 'complete' as const } } } +function getNotePath(note: NoteDoc): string { + const path = note.folderPathname === '/' ? '' : note.folderPathname + const name = note.title === '' ? 'Untitled' : note.title + return `${path}/${name}` +} + function getPromoCode(teamId: string) { return registerPromo(teamId, 'migration') .then((code) => (code.active ? code.code : undefined)) .catch(() => undefined) } +function isRetryable(response: Response) { + return ( + response.status === 408 || response.status < 400 || 499 < response.status + ) +} + async function loadFile(data: AttachmentData, name: string) { switch (data.type) { case 'blob':