From 46c524caa6fb79daa297edb2207e5fa43a623df7 Mon Sep 17 00:00:00 2001 From: Eric Anderson Date: Mon, 23 Nov 2020 11:31:31 -0800 Subject: [PATCH 01/52] wip --- localtypings/projectheader.d.ts | 14 ++- pxteditor/editor.ts | 5 +- pxteditor/workspace.ts | 9 +- webapp/src/app.tsx | 73 ++---------- webapp/src/auth.ts | 10 +- webapp/src/cloud.ts | 196 ++++++++++++++++++++++++++++++++ webapp/src/cloudsync.ts | 64 ++++++----- webapp/src/cloudworkspace.ts | 141 ----------------------- webapp/src/mkcdProvider.tsx | 35 ------ webapp/src/package.ts | 4 +- webapp/src/projects.tsx | 11 +- webapp/src/scriptmanager.tsx | 9 +- webapp/src/workspace.ts | 58 +++------- 13 files changed, 289 insertions(+), 340 deletions(-) create mode 100644 webapp/src/cloud.ts delete mode 100644 webapp/src/cloudworkspace.ts delete mode 100644 webapp/src/mkcdProvider.tsx diff --git a/localtypings/projectheader.d.ts b/localtypings/projectheader.d.ts index 07d8191e85a..38bbb80f17f 100644 --- a/localtypings/projectheader.d.ts +++ b/localtypings/projectheader.d.ts @@ -28,7 +28,8 @@ declare namespace pxt.workspace { tutorialCompleted?: pxt.tutorial.TutorialCompletionInfo; // workspace guid of the extension under test extensionUnderTest?: string; - cloudSync?: boolean; // Mark a header for syncing with a cloud provider + // id of cloud user who created this project + cloudUserId?: string; } export interface Header extends InstallHeader { @@ -41,10 +42,13 @@ declare namespace pxt.workspace { isDeleted: boolean; // mark whether or not a header has been deleted saveId?: any; // used to determine whether a project has been edited while we're saving to cloud - // For cloud providers - blobId: string; // id of the cloud blob holding this script - blobVersion: string; // version of the cloud blob - blobCurrent: boolean; // has the current version of the script been pushed to cloud + // For cloud providers -- DEPRECATED + blobId_: string; // id of the cloud blob holding this script + blobVersion_: string; // version of the cloud blob + blobCurrent_: boolean; // has the current version of the script been pushed to cloud + + cloudVersion: string; // The cloud-assigned version number (e.g. etag) + cloudCurrent: boolean; // Has the current version of the project been pushed to cloud // Used for Updating projects backupRef?: string; // guid of backed-up project (present if an update was interrupted) diff --git a/pxteditor/editor.ts b/pxteditor/editor.ts index d28a39751b6..518d94b71c2 100644 --- a/pxteditor/editor.ts +++ b/pxteditor/editor.ts @@ -215,12 +215,11 @@ namespace pxt.editor { importExampleAsync(options: ExampleImportOptions): Promise; showScriptManager(): void; importProjectDialog(): void; - cloudSync(): boolean; - cloudSignInDialog(): void; - cloudSignOut(): void; removeProject(): void; editText(): void; + hasCloudSync(): boolean; + getPreferredEditor(): string; saveAndCompile(): void; updateHeaderName(name: string): void; diff --git a/pxteditor/workspace.ts b/pxteditor/workspace.ts index 1202ac5aeab..ad62a9ce958 100644 --- a/pxteditor/workspace.ts +++ b/pxteditor/workspace.ts @@ -54,9 +54,12 @@ namespace pxt.workspace { id: U.guidGen(), recentUse: modTime, modificationTime: modTime, - blobId: null, - blobVersion: null, - blobCurrent: false, + blobId_: null, + blobVersion_: null, + blobCurrent_: false, + cloudUserId: null, + cloudCurrent: false, + cloudVersion: null, isDeleted: false, } return header diff --git a/webapp/src/app.tsx b/webapp/src/app.tsx index ab9da57404d..b0bedf39564 100644 --- a/webapp/src/app.tsx +++ b/webapp/src/app.tsx @@ -95,7 +95,7 @@ function setEditor(editor: ProjectView) { } export class ProjectView - extends data.Component + extends auth.Component implements IProjectView { editor: srceditor.Editor; editorFile: pkg.File; @@ -173,7 +173,6 @@ export class ProjectView this.openDeviceSerial = this.openDeviceSerial.bind(this); this.toggleGreenScreen = this.toggleGreenScreen.bind(this); this.toggleSimulatorFullscreen = this.toggleSimulatorFullscreen.bind(this); - this.cloudSignInComplete = this.cloudSignInComplete.bind(this); this.toggleSimulatorCollapse = this.toggleSimulatorCollapse.bind(this); this.showKeymap = this.showKeymap.bind(this); this.toggleKeymap = this.toggleKeymap.bind(this); @@ -1335,7 +1334,10 @@ export class ProjectView if (editorState.searchBar === undefined) editorState.searchBar = oldEditorState.searchBar; } - if (!h.cloudSync && this.cloudSync()) h.cloudSync = true; + // If user is signed in, sync this project to the cloud. + if (this.hasCloudSync()) { + h.cloudUserId = this.getUser()?.id; + } return compiler.newProjectAsync() .then(() => h.backupRef ? workspace.restoreFromBackupAsync(h) : Promise.resolve()) @@ -2006,62 +2008,6 @@ export class ProjectView }) } - /////////////////////////////////////////////////////////// - //////////// Cloud //////////// - /////////////////////////////////////////////////////////// - - cloudSync() { - return this.hasSync(); - } - - cloudSignInDialog() { - const providers = cloudsync.providers(); - if (providers.length == 0) - return; - if (providers.length == 1) - providers[0].loginAsync().then(() => { - this.cloudSignInComplete(); - }) - else { - // TODO: Revisit in new cloud sync - //this.signInDialog.show(); - } - } - - cloudSignOut() { - core.confirmAsync({ - header: lf("Sign out"), - body: lf("You are signing out. Make sure that you commited all your changes, local projects will be deleted."), - agreeClass: "red", - agreeIcon: "sign out", - agreeLbl: lf("Sign out"), - }).then(r => { - if (r) { - const inEditor = !!this.state.header; - // Reset the cloud workspace - return workspace.resetCloudAsync() - .then(() => { - if (inEditor) { - this.openHome(); - } - if (this.home) { - this.home.forceUpdate(); - } - }) - } - return Promise.resolve(); - }); - } - - cloudSignInComplete() { - pxt.log('cloud sign in complete'); - initLogin(); - cloudsync.syncAsync() - .then(() => { - this.forceUpdate(); - }).done(); - } - /////////////////////////////////////////////////////////// //////////// Home ///////////// /////////////////////////////////////////////////////////// @@ -2251,7 +2197,7 @@ export class ProjectView pubCurrent: false, target: pxt.appTarget.id, targetVersion: pxt.appTarget.versions.target, - cloudSync: this.cloudSync(), + cloudUserId: this.getUser()?.id, temporary: options.temporary, tutorial: options.tutorial, extensionUnderTest: options.extensionUnderTest @@ -3090,6 +3036,10 @@ export class ProjectView } } + hasCloudSync() { + return this.isLoggedIn(); + } + showScriptManager() { this.scriptManagerDialog.show(); } @@ -4545,7 +4495,8 @@ document.addEventListener("DOMContentLoaded", () => { else if (isSandbox) workspace.setupWorkspace("mem"); else if (pxt.winrt.isWinRT()) workspace.setupWorkspace("uwp"); else if (pxt.BrowserUtils.isIpcRenderer()) workspace.setupWorkspace("idb"); - else if (pxt.BrowserUtils.isLocalHost() || pxt.BrowserUtils.isPxtElectron()) workspace.setupWorkspace("fs"); + //else if (pxt.BrowserUtils.isLocalHost() || pxt.BrowserUtils.isPxtElectron()) workspace.setupWorkspace("fs"); + else workspace.setupWorkspace("browser"); Promise.resolve() .then(async () => { const href = window.location.href; diff --git a/webapp/src/auth.ts b/webapp/src/auth.ts index d562faa9640..7e2bfdd7030 100644 --- a/webapp/src/auth.ts +++ b/webapp/src/auth.ts @@ -465,12 +465,18 @@ export async function initialUserPreferences(): Promise { const state = getState(); @@ -637,5 +643,5 @@ data.mountVirtualApi(MODULE, { getSync: authApiHandler }); // ClouddWorkspace must be included after we mount our virtual APIs. -import * as cloudWorkspace from "./cloudworkspace"; +import * as cloudWorkspace from "./cloud"; cloudWorkspace.init(); diff --git a/webapp/src/cloud.ts b/webapp/src/cloud.ts new file mode 100644 index 00000000000..cf8fbbefa82 --- /dev/null +++ b/webapp/src/cloud.ts @@ -0,0 +1,196 @@ +import * as core from "./core"; +import * as auth from "./auth"; +import * as ws from "./workspace"; +import * as data from "./data"; +import * as workspace from "./workspace"; + +type Version = pxt.workspace.Version; +type File = pxt.workspace.File; +type Header = pxt.workspace.Header; +type ScriptText = pxt.workspace.ScriptText; +type WorkspaceProvider = pxt.workspace.WorkspaceProvider; + +import U = pxt.Util; + +const state = { + uploadCount: 0, + downloadCount: 0 +}; + +type CloudProject = { + id: string; + header: string; + text: string; + version: string; +}; + +async function listAsync(): Promise { + return new Promise(async (resolve, reject) => { + const result = await auth.apiAsync("/api/user/project"); + if (result.success) { + const userId = auth.user()?.id; + const headers = result.resp.map(proj => { + const header = JSON.parse(proj.header); + header.cloudUserId = userId; + header.cloudVersion = proj.version; + header.cloudCurrent = true; + return header; + }); + resolve(headers); + } else { + reject(new Error(result.errmsg)); + } + }); +} + +function getAsync(h: Header): Promise { + return new Promise(async (resolve, reject) => { + const result = await auth.apiAsync(`/api/user/project/${h.id}`); + if (result.success) { + const userId = auth.user()?.id; + const project = result.resp; + const header = JSON.parse(project.header); + const text = JSON.parse(project.text); + const version = project.version; + const file: File = { + header, + text, + version + }; + file.header.cloudCurrent = true; + file.header.cloudVersion = file.version; + file.header.cloudUserId = userId; + resolve(file); + } else { + reject(new Error(result.errmsg)); + } + }); +} + +function setAsync(h: Header, prevVersion: Version, text?: ScriptText): Promise { + return new Promise(async (resolve, reject) => { + const userId = auth.user()?.id; + h.cloudUserId = userId; + h.cloudCurrent = false; + h.cloudVersion = prevVersion; + const project: CloudProject = { + id: h.id, + header: JSON.stringify(h), + text: text ? JSON.stringify(text) : undefined, + version: prevVersion + } + const result = await auth.apiAsync('/api/user/project', project); + if (result.success) { + h.cloudCurrent = true; + h.cloudVersion = result.resp; + resolve(result.resp); + } else { + // TODO: Handle reject due to version conflict + reject(new Error(result.errmsg)); + } + }); +} + +function deleteAsync(h: Header, prevVersion: Version, text?: ScriptText): Promise { + return Promise.resolve(); +} + +function resetAsync(): Promise { + return Promise.resolve(); +} + +export async function syncAsync(): Promise { + if (!auth.hasIdentity()) { return; } + if (!await auth.loggedIn()) { return; } + try { + const userId = auth.user()?.id; + // Filter to cloud-synced headers owned by the current user. + const localCloudHeaders = workspace.getHeaders(true) + .filter(h => h.cloudUserId && h.cloudUserId === userId); + const remoteHeaders = await listAsync(); + const remoteHeaderMap = U.toDictionary(remoteHeaders, h => h.id); + const tasks = localCloudHeaders.map(async (local) => { + const remote = remoteHeaderMap[local.id]; + delete remoteHeaderMap[local.id]; + if (remote) { + if (local.cloudVersion !== remote.cloudVersion) { + if (local.cloudCurrent) { + // No local changes, download latest. + const file = await getAsync(local); + workspace.saveAsync(file.header, file.text, file.version); + } else { + // Conflict. + // TODO: Figure out how to register these. + return Promise.resolve(); + } + } else { + if (local.isDeleted) { + // Delete remote copy. + //return deleteAsync(local, local.cloudVersion); + // Mark remote copy as deleted. + remote.isDeleted = true; + return setAsync(remote, null, {}); + } + if (remote.isDeleted) { + // Delete local copy. + local.isDeleted = true; + return workspace.forceSaveAsync(local, {}) + .then(() => { data.clearCache(); }) + } + if (!local.cloudCurrent) { + // Local changes need to be synced up. + const text = await workspace.getTextAsync(local.id); + return setAsync(local, local.cloudVersion, text); + } + // Nothing to do. We're up to date locally. + return Promise.resolve(); + } + } else { + // Anomaly. Local cloud synced project exists, but no record of + // it on remote. We cannot know if there's a conflict. Convert + // to a local project. + delete local.cloudUserId; + delete local.cloudVersion; + delete local.cloudCurrent; + return workspace.saveAsync(local); + } + }); + remoteHeaders.forEach(async (remote) => { + if (remoteHeaderMap[remote.id]) { + // Project exists remotely and not locally, download it. + const file = await getAsync(remote); + tasks.push(workspace.importAsync(file.header, file.text)); + } + }) + await Promise.all(tasks); + } + catch (e) { + pxt.reportException(e); + } +} + +/** + * Virtual API + */ + +const MODULE = "cloud"; +const FIELD_UPLOADING = "uploading"; +const FIELD_DOWNLOADING = "downloading"; +const FIELD_WORKING = "working"; +export const UPLOADING = `${MODULE}:${FIELD_UPLOADING}`; +export const DOWNLOADING = `${MODULE}:${FIELD_DOWNLOADING}`; +export const WORKING = `${MODULE}:${FIELD_WORKING}`; + +function cloudApiHandler(p: string): any { + switch (data.stripProtocol(p)) { + case FIELD_UPLOADING: return state.uploadCount > 0; + case FIELD_DOWNLOADING: return state.downloadCount > 0; + case WORKING: return cloudApiHandler(UPLOADING) || cloudApiHandler(DOWNLOADING); + } + return null; +} + +export function init() { + // 'cloudws' because 'cloud' protocol is already taken. + data.mountVirtualApi("cloudws", { getSync: cloudApiHandler }); +} diff --git a/webapp/src/cloudsync.ts b/webapp/src/cloudsync.ts index 6813b67550c..b52bbd78227 100644 --- a/webapp/src/cloudsync.ts +++ b/webapp/src/cloudsync.ts @@ -4,8 +4,10 @@ import * as core from "./core"; import * as pkg from "./package"; import * as ws from "./workspace"; import * as data from "./data"; +import * as cloud from "./cloud"; type Header = pxt.workspace.Header; +type File = pxt.workspace.File; import U = pxt.Util; const lf = U.lf @@ -376,7 +378,7 @@ export async function ensureGitHubTokenAsync() { } // this is generally called by the provier's loginCheck() function -export function setProvider(impl: IdentityProvider) { +function setProvider(impl: IdentityProvider) { if (impl !== currentProvider) { currentProvider = impl invalidateData(); @@ -391,12 +393,12 @@ async function syncOneUpAsync(provider: Provider, h: Header) { text = U.flatClone(text) text[HEADER_JSON] = JSON.stringify(h, null, 4) - let firstTime = h.blobId == null + let firstTime = h.blobId_ == null let info: FileInfo try { - info = await provider.uploadAsync(h.blobId, h.blobVersion, text) + info = await provider.uploadAsync(h.blobId_, h.blobVersion_, text) } catch (e) { if (e.statusCode == 409) { core.warningNotification(lf("Conflict saving {0}; please do a full cloud sync", h.name)) @@ -408,20 +410,20 @@ async function syncOneUpAsync(provider: Provider, h: Header) { pxt.debug(`synced up ${info.id}`) if (firstTime) { - h.blobId = info.id + h.blobId_ = info.id } else { - U.assert(h.blobId == info.id) + U.assert(h.blobId_ == info.id) } - h.blobVersion = info.version + h.blobVersion_ = info.version if (h.saveId === saveId) - h.blobCurrent = true + h.blobCurrent_ = true await ws.saveAsync(h, null, true) } export async function renameAsync(h: Header, newName: string) { const provider = currentProvider && currentProvider.hasSync() && currentProvider as Provider; try { - await provider.updateAsync(h.blobId, newName) + await provider.updateAsync(h.blobId_, newName) } catch (e) { } @@ -501,7 +503,7 @@ export function refreshToken() { } export function syncAsync(): Promise { - return Promise.all([githubSyncAsync(), cloudSyncAsync()]) + return Promise.all([githubSyncAsync(), cloud.syncAsync()]) .then(() => { }); } @@ -522,10 +524,10 @@ function cloudSyncAsync(): Promise { let updated: pxt.Map = {} function uninstallAsync(h: Header) { - pxt.debug(`uninstall local ${h.blobId}`) + pxt.debug(`uninstall local ${h.blobId_}`) h.isDeleted = true - h.blobVersion = "DELETED" - h.blobCurrent = false + h.blobVersion_ = "DELETED" + h.blobCurrent_ = false return ws.saveAsync(h, null, true) } @@ -533,9 +535,9 @@ function cloudSyncAsync(): Promise { // rename current script let text = await ws.getTextAsync(header.id) let newHd = await ws.duplicateAsync(header, text) - header.blobId = null - header.blobVersion = null - header.blobCurrent = false + header.blobId_ = null + header.blobVersion_ = null + header.blobCurrent_ = false await ws.saveAsync(header, text) // get the cloud version await syncDownAsync(newHd, cloudHeader) @@ -551,20 +553,20 @@ function cloudSyncAsync(): Promise { } numDown++ - U.assert(header.blobId == cloudHeader.id) + U.assert(header.blobId_ == cloudHeader.id) let blobId = cloudHeader.version - pxt.debug(`sync down ${header.blobId} - ${blobId}`) + pxt.debug(`sync down ${header.blobId_} - ${blobId}`) return provider.downloadAsync(cloudHeader.id) .catch(core.handleNetworkError) .then((resp: FileInfo) => { - U.assert(resp.id == header.blobId) + U.assert(resp.id == header.blobId_) let files = resp.content let hd = JSON.parse(files[HEADER_JSON] || "{}") as Header delete files[HEADER_JSON] - header.cloudSync = true - header.blobCurrent = true - header.blobVersion = resp.version + header.cloudUserId = '1234' + header.blobCurrent_ = true + header.blobVersion_ = resp.version // TODO copy anything else from the cloud? header.name = hd.name || header.name || "???" header.id = header.id || hd.id || U.guidGen() @@ -579,7 +581,7 @@ function cloudSyncAsync(): Promise { header.modificationTime = resp.updatedAt || U.nowSeconds() if (!header.recentUse) header.recentUse = header.modificationTime - updated[header.blobId] = 1; + updated[header.blobId_] = 1; if (!header0) return ws.importAsync(header, files, true) @@ -613,7 +615,7 @@ function cloudSyncAsync(): Promise { } function syncDeleteAsync(h: Header) { - return provider.deleteAsync(h.blobId) + return provider.deleteAsync(h.blobId_) .then(() => uninstallAsync(h)) } @@ -625,15 +627,15 @@ function cloudSyncAsync(): Promise { // Get all local headers including those that had been deleted const allScripts = ws.getHeaders(true) const cloudHeaders = U.toDictionary(entries, e => e.id) - const existingHeaders = U.toDictionary(allScripts.filter(h => h.blobId), h => h.blobId) + const existingHeaders = U.toDictionary(allScripts.filter(h => h.blobId_), h => h.blobId_) //console.log('all', allScripts); //console.log('cloud', cloudHeaders); //console.log('existing', existingHeaders); //console.log('syncthese', allScripts.filter(hd => hd.cloudSync)); // Only syncronize those that have been marked with cloudSync - let waitFor = allScripts.filter(hd => hd.cloudSync).map(hd => { - if (cloudHeaders.hasOwnProperty(hd.blobId)) { - let chd = cloudHeaders[hd.blobId] + let waitFor = allScripts.filter(hd => hd.cloudUserId).map(hd => { + if (cloudHeaders.hasOwnProperty(hd.blobId_)) { + let chd = cloudHeaders[hd.blobId_] // The script was deleted locally, delete on cloud if (hd.isDeleted) { @@ -641,8 +643,8 @@ function cloudSyncAsync(): Promise { return syncDeleteAsync(hd) } - if (chd.version == hd.blobVersion) { - if (hd.blobCurrent) { + if (chd.version == hd.blobVersion_) { + if (hd.blobCurrent_) { // nothing to do return Promise.resolve() } else { @@ -650,7 +652,7 @@ function cloudSyncAsync(): Promise { return syncUpAsync(hd) } } else { - if (hd.blobCurrent) { + if (hd.blobCurrent_) { console.log('might have synced down: ', hd.name); return syncDownAsync(hd, chd) } else { @@ -659,7 +661,7 @@ function cloudSyncAsync(): Promise { } } } else { - if (hd.blobVersion) + if (hd.blobVersion_) // this has been pushed once to the cloud - uninstall wins return uninstallAsync(hd) else { diff --git a/webapp/src/cloudworkspace.ts b/webapp/src/cloudworkspace.ts deleted file mode 100644 index 2e436c7c770..00000000000 --- a/webapp/src/cloudworkspace.ts +++ /dev/null @@ -1,141 +0,0 @@ -import * as core from "./core"; -import * as auth from "./auth"; -import * as ws from "./workspace"; -import * as data from "./data"; -import * as workspace from "./workspace"; - -type Version = pxt.workspace.Version; -type File = pxt.workspace.File; -type Header = pxt.workspace.Header; -type Project = pxt.workspace.Project; -type ScriptText = pxt.workspace.ScriptText; -type WorkspaceProvider = pxt.workspace.WorkspaceProvider; - -import U = pxt.Util; - -const state = { - uploadCount: 0, - downloadCount: 0 -}; - -type CloudProject = { - id: string; - header: string; - text: string; - version: string; -}; - -function listAsync(): Promise { - return new Promise(async (resolve, reject) => { - const result = await auth.apiAsync("/api/user/project"); - if (result.success) { - const headers = result.resp.map(proj => JSON.parse(proj.header)); - resolve(headers); - } else { - reject(new Error(result.errmsg)); - } - }); -} - -function getAsync(h: Header): Promise { - return new Promise(async (resolve, reject) => { - const result = await auth.apiAsync(`/api/user/project/${h.id}`); - if (result.success) { - const project = result.resp; - const header = JSON.parse(project.header); - const text = JSON.parse(project.text); - const version = project.version; - const file: File = { - header, - text, - version - }; - resolve(file); - } else { - reject(new Error(result.errmsg)); - } - }); -} - -function setAsync(h: Header, prevVersion: Version, text?: ScriptText): Promise { - return new Promise(async (resolve, reject) => { - const project: CloudProject = { - id: h.id, - header: JSON.stringify(h), - text: text ? JSON.stringify(text) : undefined, - version: prevVersion - } - const result = await auth.apiAsync('/api/user/project', project); - if (result.success) { - resolve(result.resp); - } else { - // TODO: Handle reject due to version conflict - reject(new Error(result.errmsg)); - } - }); -} - -function deleteAsync(h: Header, prevVersion: Version, text?: ScriptText): Promise { - return Promise.resolve(); -} - -function resetAsync(): Promise { - return Promise.resolve(); -} - -export const provider: WorkspaceProvider = { - getAsync, - setAsync, - deleteAsync, - listAsync, - resetAsync, -} - -/** - * Virtual API - */ - -const MODULE = "cloud"; -const FIELD_UPLOADING = "uploading"; -const FIELD_DOWNLOADING = "downloading"; -const FIELD_WORKING = "working"; -export const UPLOADING = `${MODULE}:${FIELD_UPLOADING}`; -export const DOWNLOADING = `${MODULE}:${FIELD_DOWNLOADING}`; -export const WORKING = `${MODULE}:${FIELD_WORKING}`; - -function cloudApiHandler(p: string): any { - switch (data.stripProtocol(p)) { - case FIELD_UPLOADING: return state.uploadCount > 0; - case FIELD_DOWNLOADING: return state.downloadCount > 0; - case WORKING: return cloudApiHandler(UPLOADING) || cloudApiHandler(DOWNLOADING); - } - return null; -} - -export function init() { - // 'cloudws' because 'cloud' protocol is already taken. - data.mountVirtualApi("cloudws", { getSync: cloudApiHandler }); - data.subscribe(userSubscriber, auth.LOGGED_IN); -} - -let prevWorkspaceType: string; - -async function updateWorkspace() { - const loggedIn = await auth.loggedIn(); - if (loggedIn) { - // TODO: Handling of 'prev' is pretty hacky. Need to improve it. - let prev = workspace.switchToCloudWorkspace(); - if (prev !== "cloud") { - prevWorkspaceType = prev; - } - await workspace.syncAsync(); - } else if (prevWorkspaceType) { - workspace.switchToWorkspace(prevWorkspaceType); - await workspace.syncAsync(); - } -} - -const userSubscriber: data.DataSubscriber = { - subscriptions: [], - onDataChanged: async () => updateWorkspace() -}; diff --git a/webapp/src/mkcdProvider.tsx b/webapp/src/mkcdProvider.tsx deleted file mode 100644 index 2332a6568ef..00000000000 --- a/webapp/src/mkcdProvider.tsx +++ /dev/null @@ -1,35 +0,0 @@ -import * as React from "react"; -import * as sui from "./sui"; -import * as core from "./core"; -import * as auth from "./auth"; -import * as data from "./data"; -import * as codecard from "./codecard"; -import * as cloudsync from "./cloudsync"; - -// TODO: We need to do auth and cloud sync through this class. - -export class Provider extends cloudsync.ProviderBase implements cloudsync.Provider { - - constructor() { - super("mkcd", lf("MakeCode"), "xicon makecode", "https://www.makecode.com"); - } - - listAsync(): Promise { - throw new Error("Method not implemented."); - } - downloadAsync(id: string): Promise { - throw new Error("Method not implemented."); - } - uploadAsync(id: string, baseVersion: string, files: pxt.Map): Promise { - throw new Error("Method not implemented."); - } - deleteAsync(id: string): Promise { - throw new Error("Method not implemented."); - } - updateAsync(id: string, newName: string): Promise { - throw new Error("Method not implemented."); - } - getUserInfoAsync(): Promise { - throw new Error("Method not implemented."); - } -} diff --git a/webapp/src/package.ts b/webapp/src/package.ts index b0d22a16928..30ea3e8eb0a 100644 --- a/webapp/src/package.ts +++ b/webapp/src/package.ts @@ -405,14 +405,14 @@ export class EditorPackage { } savePkgAsync() { - if (this.header.blobCurrent) return Promise.resolve(); + if (this.header.blobCurrent_) return Promise.resolve(); this.savingNow++; this.updateStatus(); return workspace.saveToCloudAsync(this.header) .then(() => { this.savingNow--; this.updateStatus(); - if (!this.header.blobCurrent) + if (!this.header.blobCurrent_) this.scheduleSave(); }) } diff --git a/webapp/src/projects.tsx b/webapp/src/projects.tsx index e3425395c85..bde2fec64ce 100644 --- a/webapp/src/projects.tsx +++ b/webapp/src/projects.tsx @@ -7,6 +7,7 @@ import * as sui from "./sui"; import * as core from "./core"; import * as cloudsync from "./cloudsync"; import * as auth from "./auth"; +import * as workspace from "./workspace"; import * as identity from "./identity"; import * as codecard from "./codecard" import * as carousel from "./carousel"; @@ -21,7 +22,7 @@ interface ProjectsState { selectedIndex?: number; } -export class Projects extends data.Component { +export class Projects extends auth.Component { constructor(props: ISettingsProps) { super(props) @@ -36,7 +37,6 @@ export class Projects extends data.Component { this.chgCode = this.chgCode.bind(this); this.importProject = this.importProject.bind(this); this.showScriptManager = this.showScriptManager.bind(this); - this.cloudSignIn = this.cloudSignIn.bind(this); this.setSelected = this.setSelected.bind(this); } @@ -162,11 +162,6 @@ export class Projects extends data.Component { this.props.parent.showScriptManager(); } - cloudSignIn() { - pxt.tickEvent("projects.signin", undefined, { interactiveConsent: true }); - this.props.parent.cloudSignInDialog(); - } - renderCore() { const { selectedCategory, selectedIndex } = this.state; @@ -480,7 +475,7 @@ export class ProjectsCarousel extends data.Component workspace.duplicateAsync(header, text, res)) .then(clonedHeader => { // If we're cloud synced, update the cloudSync flag - if (this.props.parent.cloudSync()) clonedHeader.cloudSync = true; + if (this.props.parent.hasCloudSync()) clonedHeader.cloudUserId = auth.user()?.id; - delete clonedHeader.blobId - delete clonedHeader.blobVersion - delete clonedHeader.blobCurrent + delete clonedHeader.blobId_ + delete clonedHeader.blobVersion_ + delete clonedHeader.blobCurrent_ return workspace.saveAsync(clonedHeader); }) diff --git a/webapp/src/workspace.ts b/webapp/src/workspace.ts index c932a2251fe..bc9f7207b7d 100644 --- a/webapp/src/workspace.ts +++ b/webapp/src/workspace.ts @@ -11,8 +11,8 @@ import * as memoryworkspace from "./memoryworkspace" import * as iframeworkspace from "./iframeworkspace" import * as cloudsync from "./cloudsync" import * as indexedDBWorkspace from "./idbworkspace"; -import * as cloudWorkspace from "./cloudworkspace"; import * as compiler from "./compiler" +import * as auth from "./auth" import U = pxt.Util; import Cloud = pxt.Cloud; @@ -25,14 +25,9 @@ type Header = pxt.workspace.Header; type ScriptText = pxt.workspace.ScriptText; type WorkspaceProvider = pxt.workspace.WorkspaceProvider; type InstallHeader = pxt.workspace.InstallHeader; +type File = pxt.workspace.File; -interface HeaderWithScript { - header: Header; - text: ScriptText; - version: pxt.workspace.Version; -} - -let allScripts: HeaderWithScript[] = []; +let allScripts: File[] = []; let headerQ = new U.PromiseQueue(); @@ -83,9 +78,6 @@ export function setupWorkspace(id: string) { case "idb": impl = indexedDBWorkspace.provider; break; - case "cloud": - impl = cloudWorkspace.provider; - break; case "browser": default: impl = browserworkspace.provider @@ -93,19 +85,6 @@ export function setupWorkspace(id: string) { } } -export function switchToCloudWorkspace(): string { - U.assert(implType !== "cloud", "workspace already cloud"); - const prevType = implType; - impl = cloudWorkspace.provider; - implType = "cloud"; - return prevType; -} - -export function switchToWorkspace(id: string) { - impl = null; - setupWorkspace(id); -} - async function switchToMemoryWorkspace(reason: string): Promise { pxt.log(`workspace: error, switching from ${implType} to memory workspace`); @@ -139,7 +118,11 @@ async function switchToMemoryWorkspace(reason: string): Promise { export function getHeaders(withDeleted = false) { maybeSyncHeadersAsync().done(); - let r = allScripts.map(e => e.header).filter(h => (withDeleted || !h.isDeleted) && !h.isBackup) + const cloudUserId = auth.user()?.id; + let r = allScripts.map(e => e.header).filter(h => + (withDeleted || !h.isDeleted) && + !h.isBackup && + (!h.cloudUserId || h.cloudUserId === cloudUserId)) r.sort((a, b) => b.recentUse - a.recentUse) return r } @@ -335,7 +318,7 @@ export function anonymousPublishAsync(h: Header, text: ScriptText, meta: ScriptM }) } -function fixupVersionAsync(e: HeaderWithScript) { +function fixupVersionAsync(e: File) { if (e.version !== undefined) return Promise.resolve() return impl.getAsync(e.header) @@ -371,7 +354,7 @@ export function saveAsync(h: Header, text?: ScriptText, isCloud?: boolean): Prom e.text = text if (!isCloud) { h.pubCurrent = false - h.blobCurrent = false + h.blobCurrent_ = false h.modificationTime = U.nowSeconds(); h.targetVersion = h.targetVersion || "0.0.0"; } @@ -380,7 +363,7 @@ export function saveAsync(h: Header, text?: ScriptText, isCloud?: boolean): Prom } // perma-delete - if (h.isDeleted && h.blobVersion == "DELETED") { + if (h.isDeleted && h.blobVersion_ == "DELETED") { let idx = allScripts.indexOf(e) U.assert(idx >= 0) allScripts.splice(idx, 1) @@ -418,7 +401,7 @@ export function saveAsync(h: Header, text?: ScriptText, isCloud?: boolean): Prom if ((text && !isCloud) || h.isDeleted) { h.pubCurrent = false; - h.blobCurrent = false; + h.blobCurrent_ = false; h.saveId = null; data.invalidate("text:" + h.id); data.invalidate("pkg-git-status:" + h.id); @@ -444,7 +427,7 @@ function computePath(h: Header) { export function importAsync(h: Header, text: ScriptText, isCloud = false) { h.path = computePath(h) - const e: HeaderWithScript = { + const e: File = { header: h, text: text, version: null @@ -1352,21 +1335,6 @@ export function saveToCloudAsync(h: Header) { return cloudsync.saveToCloudAsync(h) } -export function resetCloudAsync(): Promise { - // always sync local scripts before resetting - // remove all cloudsync or github repositories - return syncAsync().catch(e => { }) - .then(() => cloudsync.resetAsync()) - .then(() => Promise.all(allScripts.map(e => e.header).filter(h => h.cloudSync || h.githubId).map(h => { - // Remove cloud sync'ed project - h.isDeleted = true; - h.blobVersion = "DELETED"; - return forceSaveAsync(h, null, true); - }))) - .then(() => syncAsync()) - .then(() => { }); -} - // this promise is set while a sync is in progress // cleared when sync is done. let syncAsyncPromise: Promise; From cc33cf2db383018bc7c3e836e0a78bf70098801a Mon Sep 17 00:00:00 2001 From: darzu Date: Mon, 30 Nov 2020 16:02:00 -0800 Subject: [PATCH 02/52] wip --- webapp/src/package.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/webapp/src/package.ts b/webapp/src/package.ts index 30ea3e8eb0a..fc2b913360a 100644 --- a/webapp/src/package.ts +++ b/webapp/src/package.ts @@ -405,14 +405,15 @@ export class EditorPackage { } savePkgAsync() { - if (this.header.blobCurrent_) return Promise.resolve(); + console.log("savePkgAsync") // TODO @darzu: + if (this.header.cloudCurrent) return Promise.resolve(); this.savingNow++; this.updateStatus(); return workspace.saveToCloudAsync(this.header) .then(() => { this.savingNow--; this.updateStatus(); - if (!this.header.blobCurrent_) + if (!this.header.cloudCurrent) this.scheduleSave(); }) } From 010da76fc6e54273f623dd69fad69a63803c8631 Mon Sep 17 00:00:00 2001 From: darzu Date: Mon, 30 Nov 2020 19:04:09 -0800 Subject: [PATCH 03/52] show icon for cloud projects --- localtypings/pxtpackage.d.ts | 2 ++ webapp/src/codecard.tsx | 3 +++ webapp/src/projects.tsx | 5 +++++ 3 files changed, 10 insertions(+) diff --git a/localtypings/pxtpackage.d.ts b/localtypings/pxtpackage.d.ts index 4a7e0aac5c6..d940970ef01 100644 --- a/localtypings/pxtpackage.d.ts +++ b/localtypings/pxtpackage.d.ts @@ -2,6 +2,7 @@ declare namespace pxt { type CodeCardType = "file" | "example" | "codeExample" | "tutorial" | "side" | "template" | "package" | "hw" | "forumUrl" | "forumExample" | "sharedExample"; type CodeCardEditorType = "blocks" | "js" | "py"; + type CodeCardCloudState = "local" | "cloud"; interface Map { [index: string]: T; @@ -159,6 +160,7 @@ declare namespace pxt { cardType?: CodeCardType; editor?: CodeCardEditorType; otherActions?: CodeCardAction[]; + cloudState?: CodeCardCloudState; header?: string; diff --git a/webapp/src/codecard.tsx b/webapp/src/codecard.tsx index 82f2465ab15..0b667ea9254 100644 --- a/webapp/src/codecard.tsx +++ b/webapp/src/codecard.tsx @@ -118,6 +118,9 @@ export class CodeCardView extends data.Component { {card.time ?
{card.tutorialLength ?  {lf("{0}/{1}", (card.tutorialStep || 0) + 1, card.tutorialLength)} : undefined} {card.time ? {pxt.Util.timeSince(card.time)} : null} + {card.cloudState === "cloud" && + + }
: undefined} {card.extracontent || card.learnMoreUrl || card.buyUrl || card.feedbackUrl ?
diff --git a/webapp/src/projects.tsx b/webapp/src/projects.tsx index bde2fec64ce..1fa874fea4f 100644 --- a/webapp/src/projects.tsx +++ b/webapp/src/projects.tsx @@ -610,6 +610,8 @@ export class ProjectsCarousel extends data.Component ProjectsCarousel.NUM_PROJECTS_HOMESCREEN; + // TODO @darzu: determine if project is cloud project + const headersToShow = headers .filter(h => !h.tutorial?.metadata?.hideIteration) .slice(0, ProjectsCarousel.NUM_PROJECTS_HOMESCREEN); @@ -631,6 +633,8 @@ export class ProjectsCarousel extends data.Component { if (index === 1) this.latestProject = view }} @@ -642,6 +646,7 @@ export class ProjectsCarousel extends data.Component; })} {showScriptManagerCard ?
Date: Mon, 30 Nov 2020 19:04:38 -0800 Subject: [PATCH 04/52] remove debug statements --- webapp/src/package.ts | 1 - webapp/src/projects.tsx | 2 -- 2 files changed, 3 deletions(-) diff --git a/webapp/src/package.ts b/webapp/src/package.ts index fc2b913360a..0651401d35e 100644 --- a/webapp/src/package.ts +++ b/webapp/src/package.ts @@ -405,7 +405,6 @@ export class EditorPackage { } savePkgAsync() { - console.log("savePkgAsync") // TODO @darzu: if (this.header.cloudCurrent) return Promise.resolve(); this.savingNow++; this.updateStatus(); diff --git a/webapp/src/projects.tsx b/webapp/src/projects.tsx index 1fa874fea4f..910117b10ec 100644 --- a/webapp/src/projects.tsx +++ b/webapp/src/projects.tsx @@ -610,8 +610,6 @@ export class ProjectsCarousel extends data.Component ProjectsCarousel.NUM_PROJECTS_HOMESCREEN; - // TODO @darzu: determine if project is cloud project - const headersToShow = headers .filter(h => !h.tutorial?.metadata?.hideIteration) .slice(0, ProjectsCarousel.NUM_PROJECTS_HOMESCREEN); From e15f5860fa24ad3c3e7b295591b6ba01a931382a Mon Sep 17 00:00:00 2001 From: darzu Date: Thu, 3 Dec 2020 10:14:16 -0800 Subject: [PATCH 05/52] investigative changes --- localtypings/projectheader.d.ts | 1 + webapp/src/app.tsx | 1 + webapp/src/cloud.ts | 3 ++- webapp/src/cloudsync.ts | 4 +++- webapp/src/workspace.ts | 12 ++++++++++-- 5 files changed, 17 insertions(+), 4 deletions(-) diff --git a/localtypings/projectheader.d.ts b/localtypings/projectheader.d.ts index 38bbb80f17f..31740daef47 100644 --- a/localtypings/projectheader.d.ts +++ b/localtypings/projectheader.d.ts @@ -48,6 +48,7 @@ declare namespace pxt.workspace { blobCurrent_: boolean; // has the current version of the script been pushed to cloud cloudVersion: string; // The cloud-assigned version number (e.g. etag) + // TODO @darzu: "cloudCurrent" seems very bad. This is a stateful notation and it is hard to reason about whether or not this is true. cloudCurrent: boolean; // Has the current version of the project been pushed to cloud // Used for Updating projects diff --git a/webapp/src/app.tsx b/webapp/src/app.tsx index b0bedf39564..16fe08e34f5 100644 --- a/webapp/src/app.tsx +++ b/webapp/src/app.tsx @@ -4495,6 +4495,7 @@ document.addEventListener("DOMContentLoaded", () => { else if (isSandbox) workspace.setupWorkspace("mem"); else if (pxt.winrt.isWinRT()) workspace.setupWorkspace("uwp"); else if (pxt.BrowserUtils.isIpcRenderer()) workspace.setupWorkspace("idb"); + // TODO @darzu: uncomment. this disables filesystem workspace //else if (pxt.BrowserUtils.isLocalHost() || pxt.BrowserUtils.isPxtElectron()) workspace.setupWorkspace("fs"); else workspace.setupWorkspace("browser"); Promise.resolve() diff --git a/webapp/src/cloud.ts b/webapp/src/cloud.ts index cf8fbbefa82..def901f6c46 100644 --- a/webapp/src/cloud.ts +++ b/webapp/src/cloud.ts @@ -67,7 +67,8 @@ function getAsync(h: Header): Promise { }); } -function setAsync(h: Header, prevVersion: Version, text?: ScriptText): Promise { +// TODO @darzu: is it okay to export this? +export function setAsync(h: Header, prevVersion: Version, text?: ScriptText): Promise { return new Promise(async (resolve, reject) => { const userId = auth.user()?.id; h.cloudUserId = userId; diff --git a/webapp/src/cloudsync.ts b/webapp/src/cloudsync.ts index b52bbd78227..9ae7dc25ee5 100644 --- a/webapp/src/cloudsync.ts +++ b/webapp/src/cloudsync.ts @@ -513,6 +513,7 @@ function githubSyncAsync(): Promise { } function cloudSyncAsync(): Promise { + // TODO @darzu: delete? if (!currentProvider) return Promise.resolve(undefined) if (!currentProvider.hasSync()) @@ -742,7 +743,8 @@ export function loginCheck() { impl.loginCheck(); } -export function saveToCloudAsync(h: Header) { +export async function saveToCloudAsync(h: Header) { + // TODO @darzu: why is this null when saving a new local project while logged in? if (!currentProvider || !currentProvider.hasSync()) return Promise.resolve(); diff --git a/webapp/src/workspace.ts b/webapp/src/workspace.ts index bc9f7207b7d..247508fb015 100644 --- a/webapp/src/workspace.ts +++ b/webapp/src/workspace.ts @@ -13,6 +13,7 @@ import * as cloudsync from "./cloudsync" import * as indexedDBWorkspace from "./idbworkspace"; import * as compiler from "./compiler" import * as auth from "./auth" +import * as cloud from "./cloud" import U = pxt.Util; import Cloud = pxt.Cloud; @@ -437,6 +438,7 @@ export function importAsync(h: Header, text: ScriptText, isCloud = false) { } export function installAsync(h0: InstallHeader, text: ScriptText) { + // TODO @darzu: why do we "install" here? how does that relate to "import"? This is 5 years old... U.assert(h0.target == pxt.appTarget.id); const h =
h0 @@ -1330,15 +1332,21 @@ export function installByIdAsync(id: string) { }, files))) } -export function saveToCloudAsync(h: Header) { +export async function saveToCloudAsync(h: Header) { checkHeaderSession(h); - return cloudsync.saveToCloudAsync(h) + // TODO @darzu: bypass cloudsync ? + // TODO @darzu: maybe rely on "syncAsync" instead? + const text = await getTextAsync(h.id); + return cloud.setAsync(h, h.cloudVersion, text); + + // return cloudsync.saveToCloudAsync(h) } // this promise is set while a sync is in progress // cleared when sync is done. let syncAsyncPromise: Promise; export function syncAsync(): Promise { + // TODO @darzu: this function shouldn't be needed ideally pxt.debug("workspace: sync") if (syncAsyncPromise) return syncAsyncPromise; return syncAsyncPromise = impl.listAsync() From 195b4e14de82d331cbc946b15c0879361d9612bf Mon Sep 17 00:00:00 2001 From: darzu Date: Thu, 3 Dec 2020 16:03:22 -0800 Subject: [PATCH 06/52] trying out cloudworkspace again --- webapp/src/cloud.ts | 8 ++++---- webapp/src/cloudworkspace.ts | 38 ++++++++++++++++++++++++++++++++++++ webapp/src/workspace.ts | 18 +++++++++++++++++ 3 files changed, 60 insertions(+), 4 deletions(-) create mode 100644 webapp/src/cloudworkspace.ts diff --git a/webapp/src/cloud.ts b/webapp/src/cloud.ts index def901f6c46..da17059620d 100644 --- a/webapp/src/cloud.ts +++ b/webapp/src/cloud.ts @@ -24,7 +24,7 @@ type CloudProject = { version: string; }; -async function listAsync(): Promise { +export async function listAsync(): Promise { return new Promise(async (resolve, reject) => { const result = await auth.apiAsync("/api/user/project"); if (result.success) { @@ -43,7 +43,7 @@ async function listAsync(): Promise { }); } -function getAsync(h: Header): Promise { +export function getAsync(h: Header): Promise { return new Promise(async (resolve, reject) => { const result = await auth.apiAsync(`/api/user/project/${h.id}`); if (result.success) { @@ -92,11 +92,11 @@ export function setAsync(h: Header, prevVersion: Version, text?: ScriptText): Pr }); } -function deleteAsync(h: Header, prevVersion: Version, text?: ScriptText): Promise { +export function deleteAsync(h: Header, prevVersion: Version, text?: ScriptText): Promise { return Promise.resolve(); } -function resetAsync(): Promise { +export function resetAsync(): Promise { return Promise.resolve(); } diff --git a/webapp/src/cloudworkspace.ts b/webapp/src/cloudworkspace.ts new file mode 100644 index 00000000000..1a46433d176 --- /dev/null +++ b/webapp/src/cloudworkspace.ts @@ -0,0 +1,38 @@ +import * as cloud from "./cloud"; + +type WorkspaceProvider = pxt.workspace.WorkspaceProvider; + +export const provider: WorkspaceProvider = { + getAsync: cloud.getAsync, + setAsync: cloud.setAsync, + deleteAsync: cloud.deleteAsync, + listAsync: cloud.listAsync, + resetAsync: cloud.resetAsync, +} + +// TODO @darzu: do we need a subscription here? +// export function init() { +// data.subscribe(userSubscriber, auth.LOGGED_IN); +// } + +// let prevWorkspaceType: string; + +// async function updateWorkspace() { +// const loggedIn = await auth.loggedIn(); +// if (loggedIn) { +// // TODO: Handling of 'prev' is pretty hacky. Need to improve it. +// let prev = workspace.switchToCloudWorkspace(); +// if (prev !== "cloud") { +// prevWorkspaceType = prev; +// } +// await workspace.syncAsync(); +// } else if (prevWorkspaceType) { +// workspace.switchToWorkspace(prevWorkspaceType); +// await workspace.syncAsync(); +// } +// } + +// const userSubscriber: data.DataSubscriber = { +// subscriptions: [], +// onDataChanged: async () => updateWorkspace() +// }; \ No newline at end of file diff --git a/webapp/src/workspace.ts b/webapp/src/workspace.ts index 247508fb015..ed206ec65b1 100644 --- a/webapp/src/workspace.ts +++ b/webapp/src/workspace.ts @@ -14,6 +14,7 @@ import * as indexedDBWorkspace from "./idbworkspace"; import * as compiler from "./compiler" import * as auth from "./auth" import * as cloud from "./cloud" +import * as cloudWorkspace from "./cloudworkspace" import U = pxt.Util; import Cloud = pxt.Cloud; @@ -79,6 +80,8 @@ export function setupWorkspace(id: string) { case "idb": impl = indexedDBWorkspace.provider; break; + case "cloud": + impl = cloudWorkspace.provider; case "browser": default: impl = browserworkspace.provider @@ -86,6 +89,21 @@ export function setupWorkspace(id: string) { } } +// TODO @darzu: needed? +export function switchToCloudWorkspace(): string { + U.assert(implType !== "cloud", "workspace already cloud"); + const prevType = implType; + impl = cloudWorkspace.provider; + implType = "cloud"; + return prevType; +} + +// TODO @darzu: needed? +export function switchToWorkspace(id: string) { + impl = null; + setupWorkspace(id); +} + async function switchToMemoryWorkspace(reason: string): Promise { pxt.log(`workspace: error, switching from ${implType} to memory workspace`); From 20328c6ef955748aebe9d2e9f6c6cd31a4b5a48c Mon Sep 17 00:00:00 2001 From: darzu Date: Thu, 3 Dec 2020 17:21:02 -0800 Subject: [PATCH 07/52] add browserdbworkspace.ts --- webapp/src/browserdbworkspace.ts | 58 ++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 webapp/src/browserdbworkspace.ts diff --git a/webapp/src/browserdbworkspace.ts b/webapp/src/browserdbworkspace.ts new file mode 100644 index 00000000000..77f614021b5 --- /dev/null +++ b/webapp/src/browserdbworkspace.ts @@ -0,0 +1,58 @@ +import * as db from "./db"; + +type Header = pxt.workspace.Header; +type ScriptText = pxt.workspace.ScriptText; + +export interface BrowserDbWorkspaceProvider extends pxt.workspace.WorkspaceProvider { + prefix: string; +} + +export function createBrowserDbWorkspace(namespace: string): BrowserDbWorkspaceProvider { + const prefix = namespace ? namespace + "-" : "" + const headerDb = new db.Table(`${prefix}header`); + const textDb = new db.Table(`${prefix}text`); + + async function listAsync(): Promise { + return headerDb.getAllAsync() + } + async function getAsync(h: Header): Promise { + const resp = await textDb.getAsync(h.id) + return { + header: h, + text: resp.files, + version: resp._rev + } + } + async function setAsync(h: Header, prevVer: any, text?: ScriptText): Promise { + return setCoreAsync(headerDb, textDb, h, prevVer, text); + } + async function setCoreAsync(headers: db.Table, texts: db.Table, h: Header, prevVer: any, text?: ScriptText): Promise { + const retrev = await texts.setAsync({ + id: h.id, + files: text, + _rev: prevVer + }) + const rev = await headers.setAsync(h) + h._rev = rev + return retrev + } + async function deleteAsync(h: Header, prevVer: any): Promise { + await headerDb.deleteAsync(h) + await textDb.deleteAsync({ id: h.id, _rev: h._rev }) + } + async function resetAsync() { + // workspace.resetAsync already clears all tables + // TODO @darzu: I don't like that worksapce reset does that.... + return Promise.resolve(); + } + + const provider: BrowserDbWorkspaceProvider = { + prefix, + getAsync, + setAsync, + deleteAsync, + listAsync, + resetAsync, + } + return provider; +} \ No newline at end of file From 69094688450f1e6cc3392cce677e81efbe82f5aa Mon Sep 17 00:00:00 2001 From: darzu Date: Fri, 4 Dec 2020 12:18:58 -0800 Subject: [PATCH 08/52] refactor browser workspace to build off more pure table db worksapce --- webapp/src/browserdbworkspace.ts | 18 +-- webapp/src/browserworkspace.ts | 185 ++++++++++++++----------------- 2 files changed, 95 insertions(+), 108 deletions(-) diff --git a/webapp/src/browserdbworkspace.ts b/webapp/src/browserdbworkspace.ts index 77f614021b5..07a5f2f4dfe 100644 --- a/webapp/src/browserdbworkspace.ts +++ b/webapp/src/browserdbworkspace.ts @@ -3,6 +3,12 @@ import * as db from "./db"; type Header = pxt.workspace.Header; type ScriptText = pxt.workspace.ScriptText; +type TextDbEntry = { + id: string, + files?: ScriptText, + _rev: any +} + export interface BrowserDbWorkspaceProvider extends pxt.workspace.WorkspaceProvider { prefix: string; } @@ -16,7 +22,7 @@ export function createBrowserDbWorkspace(namespace: string): BrowserDbWorkspaceP return headerDb.getAllAsync() } async function getAsync(h: Header): Promise { - const resp = await textDb.getAsync(h.id) + const resp: TextDbEntry = await textDb.getAsync(h.id) return { header: h, text: resp.files, @@ -24,15 +30,13 @@ export function createBrowserDbWorkspace(namespace: string): BrowserDbWorkspaceP } } async function setAsync(h: Header, prevVer: any, text?: ScriptText): Promise { - return setCoreAsync(headerDb, textDb, h, prevVer, text); - } - async function setCoreAsync(headers: db.Table, texts: db.Table, h: Header, prevVer: any, text?: ScriptText): Promise { - const retrev = await texts.setAsync({ + const textEnt: TextDbEntry = { id: h.id, files: text, _rev: prevVer - }) - const rev = await headers.setAsync(h) + } + const retrev = await textDb.setAsync(textEnt) + const rev = await headerDb.setAsync(h) h._rev = rev return retrev } diff --git a/webapp/src/browserworkspace.ts b/webapp/src/browserworkspace.ts index cb5daef51cf..4284bc8d88b 100644 --- a/webapp/src/browserworkspace.ts +++ b/webapp/src/browserworkspace.ts @@ -1,127 +1,110 @@ -import * as db from "./db"; - -let headers: db.Table; -let texts: db.Table; +import { BrowserDbWorkspaceProvider, createBrowserDbWorkspace } from "./browserdbworkspace"; type Header = pxt.workspace.Header; type ScriptText = pxt.workspace.ScriptText; type WorkspaceProvider = pxt.workspace.WorkspaceProvider; -function migratePrefixesAsync(): Promise { - const currentVersion = pxt.semver.parse(pxt.appTarget.versions.target); - const currentMajor = currentVersion.major; - const currentDbPrefix = pxt.appTarget.appTheme.browserDbPrefixes && pxt.appTarget.appTheme.browserDbPrefixes[currentMajor]; - - if (!currentDbPrefix) { - // This version does not use a prefix for storing projects, so just use default tables - headers = new db.Table("header"); - texts = new db.Table("text"); - return Promise.resolve(); +let currentDb: BrowserDbWorkspaceProvider; +async function init() { + if (!currentDb) { + currentDb = await createAndMigrateBrowserDb(); } +} - headers = new db.Table(`${currentDbPrefix}-header`); - texts = new db.Table(`${currentDbPrefix}-text`); - - return headers.getAllAsync() - .then((allDbHeaders) => { - if (allDbHeaders.length) { - // There are already scripts using the prefix, so a migration has already happened - return Promise.resolve(); - } - - // No headers using this prefix yet, attempt to migrate headers from previous major version (or default tables) - const previousMajor = currentMajor - 1; - const previousDbPrefix = previousMajor < 0 ? "" : pxt.appTarget.appTheme.browserDbPrefixes && pxt.appTarget.appTheme.browserDbPrefixes[previousMajor]; - let previousHeaders = new db.Table("header"); - let previousTexts = new db.Table("text"); - - if (previousDbPrefix) { - previousHeaders = new db.Table(`${previousDbPrefix}-header`); - previousTexts = new db.Table(`${previousDbPrefix}-text`); - } - - const copyProject = (h: pxt.workspace.Header): Promise => { - return previousTexts.getAsync(h.id) - .then((resp) => { - // Ignore metadata of the previous script so they get re-generated for the new copy - delete (h)._id; - delete (h)._rev; - return setAsync(h, undefined, resp.files); - }); - }; +async function migrateProject(fromWs: WorkspaceProvider, newWs: WorkspaceProvider, h: pxt.workspace.Header): Promise { + const old = await fromWs.getAsync(h) + // Ignore metadata of the previous script so they get re-generated for the new copy + delete (h)._id; + delete (h)._rev; + return await newWs.setAsync(h, undefined, old.text) +}; - return previousHeaders.getAllAsync() - .then((previousHeaders: pxt.workspace.Header[]) => { - return Promise.map(previousHeaders, (h) => copyProject(h)); - }) - .then(() => { }); - }); +const getVersionedDbPrefix = (majorVersion: number) => { + return pxt.appTarget.appTheme.browserDbPrefixes && pxt.appTarget.appTheme.browserDbPrefixes[majorVersion]; } - -function listAsync(): Promise { - return migratePrefixesAsync() - .then(() => headers.getAllAsync()); +const getCurrentDbPrefix = () => { + const currentVersion = pxt.semver.parse(pxt.appTarget.versions.target); + const currentMajor = currentVersion.major; + const currentDbPrefix = getVersionedDbPrefix(currentMajor); + return currentDbPrefix } - -function getAsync(h: Header): Promise { - return texts.getAsync(h.id) - .then(resp => ({ - header: h, - text: resp.files, - version: resp._rev - })); +const getPreviousDbPrefix = () => { + // No headers using this prefix yet, attempt to migrate headers from previous major version (or default tables) + const currentVersion = pxt.semver.parse(pxt.appTarget.versions.target); + const currentMajor = currentVersion.major; + const previousMajor = currentMajor - 1; + const previousDbPrefix = previousMajor < 0 ? "" : getVersionedDbPrefix(previousMajor); + return previousDbPrefix } -function setAsync(h: Header, prevVer: any, text?: ScriptText) { - return setCoreAsync(headers, texts, h, prevVer, text); -} +async function createAndMigrateBrowserDb(): Promise { + const currentDbPrefix = getCurrentDbPrefix(); + let currDb: BrowserDbWorkspaceProvider; + if (currentDbPrefix) { + currDb = createBrowserDbWorkspace(currentDbPrefix); + } else { + // This version does not use a prefix for storing projects, so just use default tables + currDb = createBrowserDbWorkspace(""); + return currDb; + } + + const currHeaders = await currDb.listAsync() + if (currHeaders.length) { + // There are already scripts using the prefix, so a migration has already happened + return currDb; + } -function setCoreAsync(headers: db.Table, texts: db.Table, h: Header, prevVer: any, text?: ScriptText) { - let retrev = "" - return (!text ? Promise.resolve() : - texts.setAsync({ - id: h.id, - files: text, - _rev: prevVer - }).then(rev => { - retrev = rev - })) - .then(() => headers.setAsync(h)) - .then(rev => { - h._rev = rev - return retrev - }); + // Do a migration + const prevDbPrefix = getPreviousDbPrefix(); + let prevDb: BrowserDbWorkspaceProvider; + if (prevDbPrefix) { + prevDb = createBrowserDbWorkspace(prevDbPrefix); + } else { + prevDb = createBrowserDbWorkspace(""); + } + const prevHeaders = await prevDb.listAsync() + prevHeaders.forEach(h => migrateProject(prevDb, currDb, h)); + + return currDb; } -export function copyProjectToLegacyEditor(h: Header, majorVersion: number): Promise
{ - const prefix = pxt.appTarget.appTheme.browserDbPrefixes && pxt.appTarget.appTheme.browserDbPrefixes[majorVersion]; +export async function copyProjectToLegacyEditor(h: Header, majorVersion: number): Promise
{ + await init(); - const oldHeaders = new db.Table(prefix ? `${prefix}-header` : `header`); - const oldTexts = new db.Table(prefix ? `${prefix}-text` : `text`); + const prefix = getVersionedDbPrefix(majorVersion); + const oldDb = createBrowserDbWorkspace(prefix || ""); + // clone header const header = pxt.Util.clone(h); delete (header as any)._id; delete header._rev; header.id = pxt.Util.guidGen(); - return getAsync(h) - .then(resp => setCoreAsync(oldHeaders, oldTexts, header, undefined, resp.text)) - .then(rev => header); -} - -function deleteAsync(h: Header, prevVer: any) { - return headers.deleteAsync(h) - .then(() => texts.deleteAsync({ id: h.id, _rev: h._rev })); + const resp = await currentDb.getAsync(h) + const rev = await oldDb.setAsync(header, undefined, resp.text) + return header } -function resetAsync() { - // workspace.resetAsync already clears all tables - return Promise.resolve(); -} +// TODO @darzu: might be a better way to provide this wrapping and handle the migration export const provider: WorkspaceProvider = { - getAsync, - setAsync, - deleteAsync, - listAsync, - resetAsync, + listAsync: async () => { + await init(); + return currentDb.listAsync(); + }, + getAsync: async (h: Header) => { + await init(); + return currentDb.getAsync(h); + }, + setAsync: async (h: Header, prevVersion: pxt.workspace.Version, text?: ScriptText) => { + await init(); + return currentDb.setAsync(h, prevVersion, text); + }, + deleteAsync: async (h: Header, prevVersion: pxt.workspace.Version) => { + await init(); + return currentDb.deleteAsync(h, prevVersion); + }, + resetAsync: async () => { + await init(); + return currentDb.resetAsync(); + } } \ No newline at end of file From 1c81516ef638078fdaac83c19a678101d603c0c8 Mon Sep 17 00:00:00 2001 From: darzu Date: Mon, 7 Dec 2020 09:18:39 -0800 Subject: [PATCH 09/52] error TOOO --- webapp/src/package.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/webapp/src/package.ts b/webapp/src/package.ts index 0651401d35e..09022788cca 100644 --- a/webapp/src/package.ts +++ b/webapp/src/package.ts @@ -637,6 +637,7 @@ class Host return Promise.resolve(); } if (!scr) // this should not happen; + // TODO @darzu: this is happening. return Promise.reject(new Error(`Cannot find text for package '${arg}' in the workspace.`)); if (epkg.isTopLevel() && epkg.header) return workspace.recomputeHeaderFlagsAsync(epkg.header, scr) From b19911d66955479cbe8c68ae0e8f0d05d9f4709e Mon Sep 17 00:00:00 2001 From: darzu Date: Mon, 7 Dec 2020 15:14:00 -0800 Subject: [PATCH 10/52] debug logging --- webapp/src/browserdbworkspace.ts | 23 ++++++++++++++++++++++- webapp/src/cloud.ts | 6 ++++++ webapp/src/workspace.ts | 8 ++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/webapp/src/browserdbworkspace.ts b/webapp/src/browserdbworkspace.ts index 07a5f2f4dfe..71f6f5750d2 100644 --- a/webapp/src/browserdbworkspace.ts +++ b/webapp/src/browserdbworkspace.ts @@ -18,8 +18,21 @@ export function createBrowserDbWorkspace(namespace: string): BrowserDbWorkspaceP const headerDb = new db.Table(`${prefix}header`); const textDb = new db.Table(`${prefix}text`); + // TODO @darzu: debug logging + console.log(`createBrowserDbWorkspace: ${prefix}`); + (async () => { + const hdrs: pxt.workspace.Header[] = await headerDb.getAllAsync(); + const txts: TextDbEntry[] = await textDb.getAllAsync(); + console.dir(hdrs) + console.dir(txts) + })(); + async function listAsync(): Promise { - return headerDb.getAllAsync() + const hdrs: pxt.workspace.Header[] = await headerDb.getAllAsync() + // TODO @darzu: debug logging + console.log("browser db headers:") + console.dir(hdrs.map(h => h.id)) + return hdrs } async function getAsync(h: Header): Promise { const resp: TextDbEntry = await textDb.getAsync(h.id) @@ -30,6 +43,14 @@ export function createBrowserDbWorkspace(namespace: string): BrowserDbWorkspaceP } } async function setAsync(h: Header, prevVer: any, text?: ScriptText): Promise { + // TODO @darzu: debug logging + if (!text) { + console.log("setAsync without text!") + console.dir(h) + } else { + console.log("setAsync with text :)") + } + const textEnt: TextDbEntry = { id: h.id, files: text, diff --git a/webapp/src/cloud.ts b/webapp/src/cloud.ts index da17059620d..b46ac0f66d6 100644 --- a/webapp/src/cloud.ts +++ b/webapp/src/cloud.ts @@ -26,6 +26,8 @@ type CloudProject = { export async function listAsync(): Promise { return new Promise(async (resolve, reject) => { + // TODO @darzu: this is causing errors? + console.log("listAsync"); const result = await auth.apiAsync("/api/user/project"); if (result.success) { const userId = auth.user()?.id; @@ -80,6 +82,8 @@ export function setAsync(h: Header, prevVersion: Version, text?: ScriptText): Pr text: text ? JSON.stringify(text) : undefined, version: prevVersion } + // TODO @darzu: + console.log("setAsync") const result = await auth.apiAsync('/api/user/project', project); if (result.success) { h.cloudCurrent = true; @@ -108,6 +112,8 @@ export async function syncAsync(): Promise { // Filter to cloud-synced headers owned by the current user. const localCloudHeaders = workspace.getHeaders(true) .filter(h => h.cloudUserId && h.cloudUserId === userId); + // TODO @darzu: + console.log("syncAsync: listAsync"); const remoteHeaders = await listAsync(); const remoteHeaderMap = U.toDictionary(remoteHeaders, h => h.id); const tasks = localCloudHeaders.map(async (local) => { diff --git a/webapp/src/workspace.ts b/webapp/src/workspace.ts index ed206ec65b1..7bb892fae7a 100644 --- a/webapp/src/workspace.ts +++ b/webapp/src/workspace.ts @@ -81,6 +81,8 @@ export function setupWorkspace(id: string) { impl = indexedDBWorkspace.provider; break; case "cloud": + // TODO @darzu: + console.log("CHOOSING CLOUD"); impl = cloudWorkspace.provider; case "browser": default: @@ -93,6 +95,8 @@ export function setupWorkspace(id: string) { export function switchToCloudWorkspace(): string { U.assert(implType !== "cloud", "workspace already cloud"); const prevType = implType; + // TODO @darzu: + console.log("switchToCloudWorkspace") impl = cloudWorkspace.provider; implType = "cloud"; return prevType; @@ -1352,9 +1356,13 @@ export function installByIdAsync(id: string) { export async function saveToCloudAsync(h: Header) { checkHeaderSession(h); + if (!await auth.loggedIn()) + return; // TODO @darzu: bypass cloudsync ? // TODO @darzu: maybe rely on "syncAsync" instead? const text = await getTextAsync(h.id); + // TODO @darzu: debug logging + console.log("saveToCloudAsync") return cloud.setAsync(h, h.cloudVersion, text); // return cloudsync.saveToCloudAsync(h) From c975d5de4b5cccd61078868f986b004d91ea17ad Mon Sep 17 00:00:00 2001 From: darzu Date: Mon, 7 Dec 2020 15:30:17 -0800 Subject: [PATCH 11/52] create workspaces/ folder --- webapp/src/app.tsx | 4 ++-- webapp/src/blocks.tsx | 2 +- webapp/src/cloud.ts | 5 ++--- webapp/src/compiler.ts | 2 +- webapp/src/container.tsx | 2 +- webapp/src/dialogs.tsx | 2 +- webapp/src/githubbutton.tsx | 4 ++-- webapp/src/gitjson.tsx | 4 ++-- webapp/src/identity.tsx | 2 +- webapp/src/monaco.tsx | 2 +- webapp/src/monacoFlyout.tsx | 2 +- webapp/src/package.ts | 2 +- webapp/src/projects.tsx | 4 ++-- webapp/src/screenshot.ts | 2 +- webapp/src/scriptmanager.tsx | 2 +- webapp/src/scriptsearch.tsx | 2 +- webapp/src/user.tsx | 2 +- .../src/{ => workspaces}/browserdbworkspace.ts | 2 +- .../src/{ => workspaces}/browserworkspace.ts | 0 webapp/src/{ => workspaces}/cloudsync.ts | 8 ++++---- webapp/src/{ => workspaces}/cloudworkspace.ts | 2 +- webapp/src/{ => workspaces}/fileworkspace.ts | 4 ++-- webapp/src/{ => workspaces}/githubprovider.tsx | 6 +++--- webapp/src/{ => workspaces}/googledrive.ts | 2 +- webapp/src/{ => workspaces}/idbworkspace.ts | 0 webapp/src/{ => workspaces}/iframeworkspace.ts | 0 webapp/src/{ => workspaces}/memoryworkspace.ts | 0 webapp/src/{ => workspaces}/onedrive.ts | 4 ++-- webapp/src/{ => workspaces}/workspace.ts | 18 +++++++++--------- 29 files changed, 45 insertions(+), 46 deletions(-) rename webapp/src/{ => workspaces}/browserdbworkspace.ts (98%) rename webapp/src/{ => workspaces}/browserworkspace.ts (100%) rename webapp/src/{ => workspaces}/cloudsync.ts (99%) rename webapp/src/{ => workspaces}/cloudworkspace.ts (96%) rename webapp/src/{ => workspaces}/fileworkspace.ts (98%) rename webapp/src/{ => workspaces}/githubprovider.tsx (98%) rename webapp/src/{ => workspaces}/googledrive.ts (99%) rename webapp/src/{ => workspaces}/idbworkspace.ts (100%) rename webapp/src/{ => workspaces}/iframeworkspace.ts (100%) rename webapp/src/{ => workspaces}/memoryworkspace.ts (100%) rename webapp/src/{ => workspaces}/onedrive.ts (99%) rename webapp/src/{ => workspaces}/workspace.ts (99%) diff --git a/webapp/src/app.tsx b/webapp/src/app.tsx index 16fe08e34f5..995df7133ba 100644 --- a/webapp/src/app.tsx +++ b/webapp/src/app.tsx @@ -6,8 +6,8 @@ import * as React from "react"; import * as ReactDOM from "react-dom"; -import * as workspace from "./workspace"; -import * as cloudsync from "./cloudsync"; +import * as workspace from "./workspaces/workspace"; +import * as cloudsync from "./workspaces/cloudsync"; import * as data from "./data"; import * as pkg from "./package"; import * as core from "./core"; diff --git a/webapp/src/blocks.tsx b/webapp/src/blocks.tsx index a03b96b3549..d6efd69d78b 100644 --- a/webapp/src/blocks.tsx +++ b/webapp/src/blocks.tsx @@ -8,7 +8,7 @@ import * as toolboxeditor from "./toolboxeditor" import * as compiler from "./compiler" import * as toolbox from "./toolbox"; import * as snippets from "./blocksSnippets"; -import * as workspace from "./workspace"; +import * as workspace from "./workspaces/workspace"; import * as simulator from "./simulator"; import * as dialogs from "./dialogs"; import * as blocklyFieldView from "./blocklyFieldView"; diff --git a/webapp/src/cloud.ts b/webapp/src/cloud.ts index b46ac0f66d6..61c718e8752 100644 --- a/webapp/src/cloud.ts +++ b/webapp/src/cloud.ts @@ -1,14 +1,13 @@ import * as core from "./core"; import * as auth from "./auth"; -import * as ws from "./workspace"; +import * as ws from "./workspaces/workspace"; import * as data from "./data"; -import * as workspace from "./workspace"; +import * as workspace from "./workspaces/workspace"; type Version = pxt.workspace.Version; type File = pxt.workspace.File; type Header = pxt.workspace.Header; type ScriptText = pxt.workspace.ScriptText; -type WorkspaceProvider = pxt.workspace.WorkspaceProvider; import U = pxt.Util; diff --git a/webapp/src/compiler.ts b/webapp/src/compiler.ts index 09a6bf1e5f1..964e4f5db7e 100644 --- a/webapp/src/compiler.ts +++ b/webapp/src/compiler.ts @@ -1,6 +1,6 @@ import * as pkg from "./package"; import * as core from "./core"; -import * as workspace from "./workspace"; +import * as workspace from "./workspaces/workspace"; import U = pxt.Util; diff --git a/webapp/src/container.tsx b/webapp/src/container.tsx index 2822387703d..4a08640a325 100644 --- a/webapp/src/container.tsx +++ b/webapp/src/container.tsx @@ -8,7 +8,7 @@ import * as container from "./container"; import * as core from "./core"; import * as auth from "./auth"; import * as identity from "./identity"; -import * as cloudsync from "./cloudsync"; +import * as cloudsync from "./workspaces/cloudsync"; import * as pkg from "./package"; type ISettingsProps = pxt.editor.ISettingsProps; diff --git a/webapp/src/dialogs.tsx b/webapp/src/dialogs.tsx index 9e723800e97..d859123998f 100644 --- a/webapp/src/dialogs.tsx +++ b/webapp/src/dialogs.tsx @@ -5,7 +5,7 @@ import * as sui from "./sui"; import * as core from "./core"; import * as coretsx from "./coretsx"; import * as pkg from "./package"; -import * as cloudsync from "./cloudsync"; +import * as cloudsync from "./workspaces/cloudsync"; import Cloud = pxt.Cloud; import Util = pxt.Util; diff --git a/webapp/src/githubbutton.tsx b/webapp/src/githubbutton.tsx index de00121fd9d..797c6695589 100644 --- a/webapp/src/githubbutton.tsx +++ b/webapp/src/githubbutton.tsx @@ -1,8 +1,8 @@ import * as React from "react"; import * as sui from "./sui"; import * as pkg from "./package"; -import * as cloudsync from "./cloudsync"; -import * as workspace from "./workspace"; +import * as cloudsync from "./workspaces/cloudsync"; +import * as workspace from "./workspaces/workspace"; interface GithubButtonProps extends pxt.editor.ISettingsProps { className?: string; diff --git a/webapp/src/gitjson.tsx b/webapp/src/gitjson.tsx index a582cc90eb9..327b1b7599d 100644 --- a/webapp/src/gitjson.tsx +++ b/webapp/src/gitjson.tsx @@ -3,13 +3,13 @@ import * as pkg from "./package" import * as core from "./core" import * as srceditor from "./srceditor" import * as sui from "./sui" -import * as workspace from "./workspace"; +import * as workspace from "./workspaces/workspace"; import * as dialogs from "./dialogs"; import * as coretsx from "./coretsx"; import * as data from "./data"; import * as markedui from "./marked"; import * as compiler from "./compiler"; -import * as cloudsync from "./cloudsync"; +import * as cloudsync from "./workspaces/cloudsync"; import * as tutorial from "./tutorial"; import * as _package from "./package"; diff --git a/webapp/src/identity.tsx b/webapp/src/identity.tsx index 82a69d9a006..6e2737c7693 100644 --- a/webapp/src/identity.tsx +++ b/webapp/src/identity.tsx @@ -4,7 +4,7 @@ import * as core from "./core"; import * as auth from "./auth"; import * as data from "./data"; import * as codecard from "./codecard"; -import * as cloudsync from "./cloudsync"; +import * as cloudsync from "./workspaces/cloudsync"; type ISettingsProps = pxt.editor.ISettingsProps; diff --git a/webapp/src/monaco.tsx b/webapp/src/monaco.tsx index e69804ba948..1d14d72ae43 100644 --- a/webapp/src/monaco.tsx +++ b/webapp/src/monaco.tsx @@ -12,7 +12,7 @@ import * as snippets from "./monacoSnippets" import * as pyhelper from "./monacopyhelper"; import * as simulator from "./simulator"; import * as toolbox from "./toolbox"; -import * as workspace from "./workspace"; +import * as workspace from "./workspaces/workspace"; import * as blocklyFieldView from "./blocklyFieldView"; import { ViewZoneEditorHost, ModalEditorHost, FieldEditorManager } from "./monacoFieldEditorHost"; import * as data from "./data"; diff --git a/webapp/src/monacoFlyout.tsx b/webapp/src/monacoFlyout.tsx index ee3c87a1bbb..980b777d416 100644 --- a/webapp/src/monacoFlyout.tsx +++ b/webapp/src/monacoFlyout.tsx @@ -2,7 +2,7 @@ import * as React from "react"; import * as compiler from "./compiler" import * as core from "./core"; import * as toolbox from "./toolbox"; -import * as workspace from "./workspace"; +import * as workspace from "./workspaces/workspace"; import * as data from "./data"; import * as auth from "./auth"; diff --git a/webapp/src/package.ts b/webapp/src/package.ts index 09022788cca..d7325932866 100644 --- a/webapp/src/package.ts +++ b/webapp/src/package.ts @@ -1,4 +1,4 @@ -import * as workspace from "./workspace"; +import * as workspace from "./workspaces/workspace"; import * as data from "./data"; import * as core from "./core"; import * as db from "./db"; diff --git a/webapp/src/projects.tsx b/webapp/src/projects.tsx index 910117b10ec..c0145ecf7f8 100644 --- a/webapp/src/projects.tsx +++ b/webapp/src/projects.tsx @@ -5,9 +5,9 @@ import * as ReactDOM from "react-dom"; import * as data from "./data"; import * as sui from "./sui"; import * as core from "./core"; -import * as cloudsync from "./cloudsync"; +import * as cloudsync from "./workspaces/cloudsync"; import * as auth from "./auth"; -import * as workspace from "./workspace"; +import * as workspace from "./workspaces/workspace"; import * as identity from "./identity"; import * as codecard from "./codecard" import * as carousel from "./carousel"; diff --git a/webapp/src/screenshot.ts b/webapp/src/screenshot.ts index 4f03fcf14e2..7661b17c5e6 100644 --- a/webapp/src/screenshot.ts +++ b/webapp/src/screenshot.ts @@ -1,4 +1,4 @@ -import * as workspace from "./workspace"; +import * as workspace from "./workspaces/workspace"; import * as data from "./data"; type Header = pxt.workspace.Header; diff --git a/webapp/src/scriptmanager.tsx b/webapp/src/scriptmanager.tsx index c490d2a0214..11d39cc0eb7 100644 --- a/webapp/src/scriptmanager.tsx +++ b/webapp/src/scriptmanager.tsx @@ -2,7 +2,7 @@ import * as React from "react"; import * as data from "./data"; import * as sui from "./sui"; import * as core from "./core"; -import * as workspace from "./workspace"; +import * as workspace from "./workspaces/workspace"; import * as compiler from "./compiler"; import * as auth from "./auth"; diff --git a/webapp/src/scriptsearch.tsx b/webapp/src/scriptsearch.tsx index 3dc8186d71d..3559fbcb81f 100644 --- a/webapp/src/scriptsearch.tsx +++ b/webapp/src/scriptsearch.tsx @@ -8,7 +8,7 @@ import * as pkg from "./package"; import * as core from "./core"; import * as codecard from "./codecard"; import * as electron from "./electron"; -import * as workspace from "./workspace"; +import * as workspace from "./workspaces/workspace"; import { SearchInput } from "./components/searchInput"; type ISettingsProps = pxt.editor.ISettingsProps; diff --git a/webapp/src/user.tsx b/webapp/src/user.tsx index 36be1bf71cb..d29b446922a 100644 --- a/webapp/src/user.tsx +++ b/webapp/src/user.tsx @@ -2,7 +2,7 @@ import * as React from "react"; import * as sui from "./sui"; import * as core from "./core"; import * as auth from "./auth"; -import * as cloudsync from "./cloudsync"; +import * as cloudsync from "./workspaces/cloudsync"; type ISettingsProps = pxt.editor.ISettingsProps; diff --git a/webapp/src/browserdbworkspace.ts b/webapp/src/workspaces/browserdbworkspace.ts similarity index 98% rename from webapp/src/browserdbworkspace.ts rename to webapp/src/workspaces/browserdbworkspace.ts index 71f6f5750d2..80f645bf253 100644 --- a/webapp/src/browserdbworkspace.ts +++ b/webapp/src/workspaces/browserdbworkspace.ts @@ -1,4 +1,4 @@ -import * as db from "./db"; +import * as db from "../db"; type Header = pxt.workspace.Header; type ScriptText = pxt.workspace.ScriptText; diff --git a/webapp/src/browserworkspace.ts b/webapp/src/workspaces/browserworkspace.ts similarity index 100% rename from webapp/src/browserworkspace.ts rename to webapp/src/workspaces/browserworkspace.ts diff --git a/webapp/src/cloudsync.ts b/webapp/src/workspaces/cloudsync.ts similarity index 99% rename from webapp/src/cloudsync.ts rename to webapp/src/workspaces/cloudsync.ts index 9ae7dc25ee5..3ea3d4e7085 100644 --- a/webapp/src/cloudsync.ts +++ b/webapp/src/workspaces/cloudsync.ts @@ -1,10 +1,10 @@ // TODO cloud save indication in the editor somewhere -import * as core from "./core"; -import * as pkg from "./package"; +import * as core from "../core"; +import * as pkg from "../package"; import * as ws from "./workspace"; -import * as data from "./data"; -import * as cloud from "./cloud"; +import * as data from "../data"; +import * as cloud from "../cloud"; type Header = pxt.workspace.Header; type File = pxt.workspace.File; diff --git a/webapp/src/cloudworkspace.ts b/webapp/src/workspaces/cloudworkspace.ts similarity index 96% rename from webapp/src/cloudworkspace.ts rename to webapp/src/workspaces/cloudworkspace.ts index 1a46433d176..8c5914aca6b 100644 --- a/webapp/src/cloudworkspace.ts +++ b/webapp/src/workspaces/cloudworkspace.ts @@ -1,4 +1,4 @@ -import * as cloud from "./cloud"; +import * as cloud from "../cloud"; type WorkspaceProvider = pxt.workspace.WorkspaceProvider; diff --git a/webapp/src/fileworkspace.ts b/webapp/src/workspaces/fileworkspace.ts similarity index 98% rename from webapp/src/fileworkspace.ts rename to webapp/src/workspaces/fileworkspace.ts index 0230ba2fdce..7214eac34b0 100644 --- a/webapp/src/fileworkspace.ts +++ b/webapp/src/workspaces/fileworkspace.ts @@ -1,5 +1,5 @@ -import * as core from "./core"; -import * as electron from "./electron"; +import * as core from "../core"; +import * as electron from "../electron"; import U = pxt.Util; import Cloud = pxt.Cloud; diff --git a/webapp/src/githubprovider.tsx b/webapp/src/workspaces/githubprovider.tsx similarity index 98% rename from webapp/src/githubprovider.tsx rename to webapp/src/workspaces/githubprovider.tsx index 3fd265d5dc5..f9821534056 100644 --- a/webapp/src/githubprovider.tsx +++ b/webapp/src/workspaces/githubprovider.tsx @@ -1,8 +1,8 @@ import * as React from "react"; -import * as sui from "./sui"; -import * as core from "./core"; +import * as sui from "../sui"; +import * as core from "../core"; import * as cloudsync from "./cloudsync"; -import * as dialogs from "./dialogs"; +import * as dialogs from "../dialogs"; import * as workspace from "./workspace"; export const PROVIDER_NAME = "github"; diff --git a/webapp/src/googledrive.ts b/webapp/src/workspaces/googledrive.ts similarity index 99% rename from webapp/src/googledrive.ts rename to webapp/src/workspaces/googledrive.ts index 4151a686714..c39358b799a 100644 --- a/webapp/src/googledrive.ts +++ b/webapp/src/workspaces/googledrive.ts @@ -1,4 +1,4 @@ -import * as core from "./core"; +import * as core from "../core"; import * as cloudsync from "./cloudsync"; import U = pxt.U diff --git a/webapp/src/idbworkspace.ts b/webapp/src/workspaces/idbworkspace.ts similarity index 100% rename from webapp/src/idbworkspace.ts rename to webapp/src/workspaces/idbworkspace.ts diff --git a/webapp/src/iframeworkspace.ts b/webapp/src/workspaces/iframeworkspace.ts similarity index 100% rename from webapp/src/iframeworkspace.ts rename to webapp/src/workspaces/iframeworkspace.ts diff --git a/webapp/src/memoryworkspace.ts b/webapp/src/workspaces/memoryworkspace.ts similarity index 100% rename from webapp/src/memoryworkspace.ts rename to webapp/src/workspaces/memoryworkspace.ts diff --git a/webapp/src/onedrive.ts b/webapp/src/workspaces/onedrive.ts similarity index 99% rename from webapp/src/onedrive.ts rename to webapp/src/workspaces/onedrive.ts index 49cf6e068a3..186fcb1e92c 100644 --- a/webapp/src/onedrive.ts +++ b/webapp/src/workspaces/onedrive.ts @@ -1,6 +1,6 @@ -import * as core from "./core"; +import * as core from "../core"; import * as cloudsync from "./cloudsync"; -import * as data from "./data"; +import * as data from "../data"; const rootdir = "/me/drive/special/approot" diff --git a/webapp/src/workspace.ts b/webapp/src/workspaces/workspace.ts similarity index 99% rename from webapp/src/workspace.ts rename to webapp/src/workspaces/workspace.ts index 7bb892fae7a..d7380685bf1 100644 --- a/webapp/src/workspace.ts +++ b/webapp/src/workspaces/workspace.ts @@ -1,19 +1,19 @@ -/// -/// -/// +/// +/// +/// -import * as db from "./db"; -import * as core from "./core"; -import * as data from "./data"; +import * as db from "../db"; +import * as core from "../core"; +import * as data from "../data"; import * as browserworkspace from "./browserworkspace" import * as fileworkspace from "./fileworkspace" import * as memoryworkspace from "./memoryworkspace" import * as iframeworkspace from "./iframeworkspace" import * as cloudsync from "./cloudsync" import * as indexedDBWorkspace from "./idbworkspace"; -import * as compiler from "./compiler" -import * as auth from "./auth" -import * as cloud from "./cloud" +import * as compiler from "../compiler" +import * as auth from "../auth" +import * as cloud from "../cloud" import * as cloudWorkspace from "./cloudworkspace" import U = pxt.Util; From 3ff57dd7354288e65087b439ecf2369041e01ed8 Mon Sep 17 00:00:00 2001 From: darzu Date: Mon, 7 Dec 2020 16:11:40 -0800 Subject: [PATCH 12/52] refactor setupWorkspace into pure chooseWorkspace --- webapp/src/workspaces/workspace.ts | 47 ++++++++++++++++-------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/webapp/src/workspaces/workspace.ts b/webapp/src/workspaces/workspace.ts index d7380685bf1..9403778ff5a 100644 --- a/webapp/src/workspaces/workspace.ts +++ b/webapp/src/workspaces/workspace.ts @@ -34,7 +34,7 @@ let allScripts: File[] = []; let headerQ = new U.PromiseQueue(); let impl: WorkspaceProvider; -let implType: string; +let implType: WorkspaceKind; function lookup(id: string) { return allScripts.find(x => x.header.id == id || x.header.path == id); @@ -55,40 +55,43 @@ export function copyProjectToLegacyEditor(header: Header, majorVersion: number): return browserworkspace.copyProjectToLegacyEditor(header, majorVersion); } -export function setupWorkspace(id: string) { - U.assert(!impl, "workspace set twice"); - pxt.log(`workspace: ${id}`); - implType = id ?? "browser"; - switch (id) { + +type WorkspaceKind = "browser" | "fs" | "file" | "mem" | "memory" | "iframe" | "uwp" | "idb" | "cloud"; + +function chooseWorkspace(kind: WorkspaceKind = "browser"): pxt.workspace.WorkspaceProvider { + switch (kind) { case "fs": case "file": // Local file workspace, serializes data under target/projects/ - impl = fileworkspace.provider; - break; + return fileworkspace.provider; case "mem": case "memory": - impl = memoryworkspace.provider; - break; + return memoryworkspace.provider; case "iframe": // Iframe workspace, the editor relays sync messages back and forth when hosted in an iframe - impl = iframeworkspace.provider; - break; + return iframeworkspace.provider; case "uwp": fileworkspace.setApiAsync(pxt.winrt.workspace.fileApiAsync); - impl = pxt.winrt.workspace.getProvider(fileworkspace.provider); - break; + return pxt.winrt.workspace.getProvider(fileworkspace.provider); case "idb": - impl = indexedDBWorkspace.provider; - break; + return indexedDBWorkspace.provider; case "cloud": // TODO @darzu: - console.log("CHOOSING CLOUD"); - impl = cloudWorkspace.provider; + console.log("CHOOSING CLOUD WORKSPACE"); + return cloudWorkspace.provider; case "browser": - default: - impl = browserworkspace.provider - break; + return browserworkspace.provider } + // exhaustivity check + const _never: never = kind; + return _never; +} + +export function setupWorkspace(kind: WorkspaceKind): void { + U.assert(!impl, "workspace set twice"); + pxt.log(`workspace: ${kind}`); + implType = kind ?? "browser"; + impl = chooseWorkspace(implType); } // TODO @darzu: needed? @@ -103,7 +106,7 @@ export function switchToCloudWorkspace(): string { } // TODO @darzu: needed? -export function switchToWorkspace(id: string) { +export function switchToWorkspace(id: WorkspaceKind) { impl = null; setupWorkspace(id); } From a5063a702012eb5192322e1ce6dfe697bbb2a8e5 Mon Sep 17 00:00:00 2001 From: darzu Date: Mon, 7 Dec 2020 16:17:38 -0800 Subject: [PATCH 13/52] export --- webapp/src/workspaces/workspace.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp/src/workspaces/workspace.ts b/webapp/src/workspaces/workspace.ts index 9403778ff5a..a10e85cbee7 100644 --- a/webapp/src/workspaces/workspace.ts +++ b/webapp/src/workspaces/workspace.ts @@ -56,7 +56,7 @@ export function copyProjectToLegacyEditor(header: Header, majorVersion: number): } -type WorkspaceKind = "browser" | "fs" | "file" | "mem" | "memory" | "iframe" | "uwp" | "idb" | "cloud"; +export type WorkspaceKind = "browser" | "fs" | "file" | "mem" | "memory" | "iframe" | "uwp" | "idb" | "cloud"; function chooseWorkspace(kind: WorkspaceKind = "browser"): pxt.workspace.WorkspaceProvider { switch (kind) { From 423fe0ac09505fc266cae73d4e146005c6341a8e Mon Sep 17 00:00:00 2001 From: darzu Date: Tue, 8 Dec 2020 10:53:27 -0800 Subject: [PATCH 14/52] cloud debug logging --- webapp/src/cloud.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/webapp/src/cloud.ts b/webapp/src/cloud.ts index 61c718e8752..9e0ce175dab 100644 --- a/webapp/src/cloud.ts +++ b/webapp/src/cloud.ts @@ -26,7 +26,7 @@ type CloudProject = { export async function listAsync(): Promise { return new Promise(async (resolve, reject) => { // TODO @darzu: this is causing errors? - console.log("listAsync"); + console.log("cloud.ts:listAsync"); const result = await auth.apiAsync("/api/user/project"); if (result.success) { const userId = auth.user()?.id; @@ -45,6 +45,7 @@ export async function listAsync(): Promise { } export function getAsync(h: Header): Promise { + console.log(`cloud.ts:getAsync ${h.id}`); // TODO @darzu: return new Promise(async (resolve, reject) => { const result = await auth.apiAsync(`/api/user/project/${h.id}`); if (result.success) { @@ -70,6 +71,7 @@ export function getAsync(h: Header): Promise { // TODO @darzu: is it okay to export this? export function setAsync(h: Header, prevVersion: Version, text?: ScriptText): Promise { + console.log(`cloud.ts:setAsync ${h.id}`); // TODO @darzu: return new Promise(async (resolve, reject) => { const userId = auth.user()?.id; h.cloudUserId = userId; @@ -96,23 +98,24 @@ export function setAsync(h: Header, prevVersion: Version, text?: ScriptText): Pr } export function deleteAsync(h: Header, prevVersion: Version, text?: ScriptText): Promise { + console.log(`cloud.ts:deleteAsync ${h.id}`); // TODO @darzu: return Promise.resolve(); } export function resetAsync(): Promise { + console.log(`cloud.ts:resetAsync`); // TODO @darzu: return Promise.resolve(); } export async function syncAsync(): Promise { if (!auth.hasIdentity()) { return; } if (!await auth.loggedIn()) { return; } + console.log(`cloud.ts:syncAsync`); // TODO @darzu: try { const userId = auth.user()?.id; // Filter to cloud-synced headers owned by the current user. const localCloudHeaders = workspace.getHeaders(true) .filter(h => h.cloudUserId && h.cloudUserId === userId); - // TODO @darzu: - console.log("syncAsync: listAsync"); const remoteHeaders = await listAsync(); const remoteHeaderMap = U.toDictionary(remoteHeaders, h => h.id); const tasks = localCloudHeaders.map(async (local) => { From 7c732310c1c94bb4020e570e808af72a318b8b3d Mon Sep 17 00:00:00 2001 From: darzu Date: Tue, 8 Dec 2020 10:53:49 -0800 Subject: [PATCH 15/52] debugging and type cleanup --- webapp/src/app.tsx | 4 +++- webapp/src/workspaces/workspace.ts | 6 +++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/webapp/src/app.tsx b/webapp/src/app.tsx index 995df7133ba..ea0285c141f 100644 --- a/webapp/src/app.tsx +++ b/webapp/src/app.tsx @@ -4490,7 +4490,9 @@ document.addEventListener("DOMContentLoaded", () => { const isSandbox = pxt.shell.isSandboxMode() || pxt.shell.isReadOnly(); const isController = pxt.shell.isControllerMode(); const theme = pxt.appTarget.appTheme; - if (query["ws"]) workspace.setupWorkspace(query["ws"]); + if (query["ws"]) { + workspace.setupWorkspace(query["ws"] as workspace.WorkspaceKind) + } else if ((theme.allowParentController || isController) && pxt.BrowserUtils.isIFrame()) workspace.setupWorkspace("iframe"); else if (isSandbox) workspace.setupWorkspace("mem"); else if (pxt.winrt.isWinRT()) workspace.setupWorkspace("uwp"); diff --git a/webapp/src/workspaces/workspace.ts b/webapp/src/workspaces/workspace.ts index a10e85cbee7..b8bfffce583 100644 --- a/webapp/src/workspaces/workspace.ts +++ b/webapp/src/workspaces/workspace.ts @@ -80,16 +80,16 @@ function chooseWorkspace(kind: WorkspaceKind = "browser"): pxt.workspace.Workspa console.log("CHOOSING CLOUD WORKSPACE"); return cloudWorkspace.provider; case "browser": + default: return browserworkspace.provider } - // exhaustivity check - const _never: never = kind; - return _never; } export function setupWorkspace(kind: WorkspaceKind): void { U.assert(!impl, "workspace set twice"); pxt.log(`workspace: ${kind}`); + // TODO @darzu: + console.log(`choosing workspace: ${kind}`); implType = kind ?? "browser"; impl = chooseWorkspace(implType); } From 142b6c689b2b91a71e19127faf2500c75e7868a1 Mon Sep 17 00:00:00 2001 From: darzu Date: Tue, 8 Dec 2020 13:45:23 -0800 Subject: [PATCH 16/52] investigative comments --- localtypings/projectheader.d.ts | 2 ++ webapp/src/workspaces/cloudsync.ts | 7 ++++++- webapp/src/workspaces/workspace.ts | 29 +++++++++++++++++++++++++---- 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/localtypings/projectheader.d.ts b/localtypings/projectheader.d.ts index 31740daef47..bd990bf856b 100644 --- a/localtypings/projectheader.d.ts +++ b/localtypings/projectheader.d.ts @@ -41,6 +41,8 @@ declare namespace pxt.workspace { isDeleted: boolean; // mark whether or not a header has been deleted saveId?: any; // used to determine whether a project has been edited while we're saving to cloud + + // TODO @darzu: remove all of these? // For cloud providers -- DEPRECATED blobId_: string; // id of the cloud blob holding this script diff --git a/webapp/src/workspaces/cloudsync.ts b/webapp/src/workspaces/cloudsync.ts index 3ea3d4e7085..a2a643ab6ba 100644 --- a/webapp/src/workspaces/cloudsync.ts +++ b/webapp/src/workspaces/cloudsync.ts @@ -503,7 +503,12 @@ export function refreshToken() { } export function syncAsync(): Promise { - return Promise.all([githubSyncAsync(), cloud.syncAsync()]) + return Promise.all([ + githubSyncAsync(), + // TODO @darzu: + // cloud.syncAsync() + cloudSyncAsync() + ]) .then(() => { }); } diff --git a/webapp/src/workspaces/workspace.ts b/webapp/src/workspaces/workspace.ts index b8bfffce583..395723b4400 100644 --- a/webapp/src/workspaces/workspace.ts +++ b/webapp/src/workspaces/workspace.ts @@ -143,6 +143,7 @@ async function switchToMemoryWorkspace(reason: string): Promise { } export function getHeaders(withDeleted = false) { + // TODO @darzu: we need to consolidate this to one Workspace impl maybeSyncHeadersAsync().done(); const cloudUserId = auth.user()?.id; let r = allScripts.map(e => e.header).filter(h => @@ -154,6 +155,7 @@ export function getHeaders(withDeleted = false) { } export function makeBackupAsync(h: Header, text: ScriptText): Promise
{ + // TODO @darzu: check mechanism & policy backup system let h2 = U.flatClone(h) h2.id = U.guidGen() @@ -224,6 +226,7 @@ function maybeSyncHeadersAsync(): Promise { return Promise.resolve(); } function refreshHeadersSession() { + // TODO @darzu: carefully handle this // use # of scripts + time of last mod as key sessionID = allScripts.length + ' ' + allScripts .map(h => h.header.modificationTime) @@ -345,6 +348,7 @@ export function anonymousPublishAsync(h: Header, text: ScriptText, meta: ScriptM } function fixupVersionAsync(e: File) { + // TODO @darzu: need to handle one-off tasks like this if (e.version !== undefined) return Promise.resolve() return impl.getAsync(e.header) @@ -354,7 +358,7 @@ function fixupVersionAsync(e: File) { } export function forceSaveAsync(h: Header, text?: ScriptText, isCloud?: boolean): Promise { - clearHeaderSession(h); + clearHeaderSession(h); // TODO @darzu: why do we conservatively call clearHeaderSession everywhere? return saveAsync(h, text, isCloud); } @@ -362,12 +366,14 @@ export function saveAsync(h: Header, text?: ScriptText, isCloud?: boolean): Prom pxt.debug(`workspace: save ${h.id}`) if (h.isDeleted) clearHeaderSession(h); - checkHeaderSession(h); + checkHeaderSession(h); // TODO @darzu: what is header session... U.assert(h.target == pxt.appTarget.id); - if (h.temporary) + if (h.temporary) { + // TODO @darzu: lol... what abstraction does this fit? return Promise.resolve() + } let e = lookup(h.id) //U.assert(e.header === h) @@ -390,6 +396,7 @@ export function saveAsync(h: Header, text?: ScriptText, isCloud?: boolean): Prom // perma-delete if (h.isDeleted && h.blobVersion_ == "DELETED") { + // TODO @darzu: "isDelete" is a command flag????? argh.. let idx = allScripts.indexOf(e) U.assert(idx >= 0) allScripts.splice(idx, 1) @@ -401,12 +408,14 @@ export function saveAsync(h: Header, text?: ScriptText, isCloud?: boolean): Prom // check if we have dynamic boards, store board info for home page rendering if (text && pxt.appTarget.simulator && pxt.appTarget.simulator.dynamicBoardDefinition) { + // TODO @darzu: what does this mean policy-wise... const pxtjson = pxt.Package.parseAndValidConfig(text[pxt.CONFIG_NAME]); if (pxtjson && pxtjson.dependencies) h.board = Object.keys(pxtjson.dependencies) .filter(p => !!pxt.bundledSvg(p))[0]; } + // TODO @darzu: what is this "headerQ" and why does it exist... return headerQ.enqueue(h.id, async () => { await fixupVersionAsync(e); let ver: any; @@ -417,6 +426,7 @@ export function saveAsync(h: Header, text?: ScriptText, isCloud?: boolean): Prom ver = await impl.setAsync(h, e.version, toWrite); } catch (e) { // Write failed; use in memory db. + // TODO @darzu: POLICY await switchToMemoryWorkspace("write failed"); ver = await impl.setAsync(h, e.version, toWrite); } @@ -429,6 +439,7 @@ export function saveAsync(h: Header, text?: ScriptText, isCloud?: boolean): Prom h.pubCurrent = false; h.blobCurrent_ = false; h.saveId = null; + // TODO @darzu: more data api syncing.. data.invalidate("text:" + h.id); data.invalidate("pkg-git-status:" + h.id); } @@ -452,6 +463,8 @@ function computePath(h: Header) { } export function importAsync(h: Header, text: ScriptText, isCloud = false) { + // TODO @darzu: why does import bypass workspaces or does it? + console.log(`importAsync: ${h.id}`); h.path = computePath(h) const e: File = { header: h, @@ -646,6 +659,7 @@ export async function hasMergeConflictMarkersAsync(hd: Header): Promise } export async function prAsync(hd: Header, commitId: string, msg: string) { + // TODO @darzu: this gh stuff should be moved elsewhere probably.. let parsed = pxt.github.parseRepoId(hd.githubId) // merge conflict - create a Pull Request const branchName = await pxt.github.getNewBranchNameAsync(parsed.fullName, "merge-") @@ -1342,6 +1356,7 @@ export function downloadFilesByIdAsync(id: string): Promise> { } export function installByIdAsync(id: string) { + // TODO @darzu: what is install? return Cloud.privateGetAsync(id, /* forceLiveEndpoint */ true) .then((scr: Cloud.JsonScript) => getPublishedScriptAsync(scr.id) @@ -1382,6 +1397,7 @@ export function syncAsync(): Promise { .catch((e) => { // There might be a problem with the native databases. Switch to memory for this session so the user can at // least use the editor. + // TODO @darzu: POLICY return switchToMemoryWorkspace("sync failed") .then(() => impl.listAsync()); }) @@ -1396,6 +1412,8 @@ export function syncAsync(): Promise { // force reload ex.text = undefined ex.version = undefined + // TODO @darzu: handle data API subscriptions on header changed + console.log(`INVALIDATIN header ${hd.id}`) // TODO @darzu: data.invalidateHeader("header", hd); data.invalidateHeader("text", hd); data.invalidateHeader("pkg-git-status", hd); @@ -1413,7 +1431,7 @@ export function syncAsync(): Promise { cloudsync.syncAsync().done() // sync in background }) .then(() => { - refreshHeadersSession(); + // TODO @darzu: what does refreshHeadersSession do? return impl.getSyncState ? impl.getSyncState() : null }) .finally(() => { @@ -1422,6 +1440,7 @@ export function syncAsync(): Promise { } export function resetAsync() { + // TODO @darzu: this should just pass through to workspace impl allScripts = [] return impl.resetAsync() .then(cloudsync.resetAsync) @@ -1491,6 +1510,7 @@ data.mountVirtualApi("headers", { p = data.stripProtocol(p) const headers = getHeaders() if (!p) return Promise.resolve(headers) + console.log(`data SEARCH headers:${p}`) // TODO @darzu: return compiler.projectSearchAsync({ term: p, headers }) .then((searchResults: pxtc.service.ProjectSearchInfo[]) => searchResults) .then(searchResults => { @@ -1511,6 +1531,7 @@ data.mountVirtualApi("headers", { data.mountVirtualApi("text", { getAsync: p => { const m = /^[\w\-]+:([^\/]+)(\/(.*))?/.exec(p) + // TODO @darzu: thin layer over workspace impl? return getTextAsync(m[1]) .then(files => { if (m[3]) From b64be99e838fef896bcd6e711a7a236378026197 Mon Sep 17 00:00:00 2001 From: darzu Date: Tue, 8 Dec 2020 13:45:39 -0800 Subject: [PATCH 17/52] starting JointWorkspace --- webapp/src/workspaces/jointworkspace.ts | 54 +++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 webapp/src/workspaces/jointworkspace.ts diff --git a/webapp/src/workspaces/jointworkspace.ts b/webapp/src/workspaces/jointworkspace.ts new file mode 100644 index 00000000000..061d3a5fde1 --- /dev/null +++ b/webapp/src/workspaces/jointworkspace.ts @@ -0,0 +1,54 @@ +type Header = pxt.workspace.Header; +type ScriptText = pxt.workspace.ScriptText; +type File = pxt.workspace.File; +type WorkspaceProvider = pxt.workspace.WorkspaceProvider; + +export function createJointWorkspace(primary: WorkspaceProvider, ...others: WorkspaceProvider[]): WorkspaceProvider { + const all: WorkspaceProvider[] = [primary, ...others]; + + // TODO @darzu: debug logging + console.log(`createJointWorkspace`); + + async function listAsync(): Promise { + const allHdrs = (await Promise.all(all.map(ws => ws.listAsync()))) + .reduce((p, n) => [...p, ...n], []) + const seenHdrs: { [key: string]: boolean } = {} + // de-duplicate headers (prefering earlier ones) + const res = allHdrs.reduce((p, n) => { + if (seenHdrs[n.id]) + return p; + seenHdrs[n.id] = true; + return [...p, n] + }, []) + return res; + } + async function getAsync(h: Header): Promise { + // chose the first matching one + return all.reduce(async (p: Promise, n) => await p ?? n.getAsync(h), null) + } + async function getWorkspaceForAsync(h: Header): Promise { + return await all.reduce( + async (p: Promise, n) => await p ?? n.getAsync(h).then(f => f ? n : null), null) + } + async function setAsync(h: Header, prevVer: any, text?: ScriptText): Promise { + const matchingWorkspace = await getWorkspaceForAsync(h) + const ws = matchingWorkspace ?? primary + return ws.setAsync(h, prevVer, text) + } + async function deleteAsync(h: Header, prevVer: any): Promise { + const matchingWorkspace = await getWorkspaceForAsync(h) + return matchingWorkspace?.deleteAsync(h, prevVer) + } + async function resetAsync() { + await Promise.all(all.map(ws => ws.resetAsync())) + } + + const provider: WorkspaceProvider = { + getAsync, + setAsync, + deleteAsync, + listAsync, + resetAsync, + } + return provider; +} \ No newline at end of file From 631f2003719117686520306cd38157ac18889df0 Mon Sep 17 00:00:00 2001 From: darzu Date: Tue, 8 Dec 2020 19:33:16 -0800 Subject: [PATCH 18/52] toying with sync workspace --- webapp/src/workspaces/syncworkspace.ts | 79 ++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 webapp/src/workspaces/syncworkspace.ts diff --git a/webapp/src/workspaces/syncworkspace.ts b/webapp/src/workspaces/syncworkspace.ts new file mode 100644 index 00000000000..a3fad9df936 --- /dev/null +++ b/webapp/src/workspaces/syncworkspace.ts @@ -0,0 +1,79 @@ + + +type Header = pxt.workspace.Header; +type ScriptText = pxt.workspace.ScriptText; +type File = pxt.workspace.File; +type WorkspaceProvider = pxt.workspace.WorkspaceProvider; +type Version = pxt.workspace.Version; +type Asset = pxt.workspace.Version; + +export interface SyncWorkspaceProvider extends WorkspaceProvider { + listSync(): Header[]; + getSync(h: Header): File; + setSync(h: Header, prevVersion: Version, text?: ScriptText): Version; + deleteSync?: (h: Header, prevVersion: Version) => void; + resetSync(): void; + loadedSync?: () => void; + getSyncState?: () => pxt.editor.EditorSyncState; + saveScreenshotSync?: (h: Header, screenshot: string, icon: string) => void; + saveAssetSync?: (id: string, filename: string, data: Uint8Array) => void; + listAssetsSync?: (id: string) => Asset[]; +} + +export function createCachedWorkspace(source: WorkspaceProvider) { + // TODO @darzu: debug logging + console.log(`createCachedWorkspace`); + + // TODO @darzu: useful? + + const provider: SyncWorkspaceProvider = { + // sync + listSync: (): Header[] => { + throw "not impl"; + }, + getSync: (h: Header): File => { + throw "not impl"; + }, + setSync: (h: Header, prevVersion: Version, text?: ScriptText): Version => { + throw "not impl"; + }, + deleteSync: (h: Header, prevVersion: Version): void => { + throw "not impl"; + }, + resetSync: (): void => { + throw "not impl"; + }, + loadedSync: (): void => { + throw "not impl"; + }, + getSyncState: (): pxt.editor.EditorSyncState => { + throw "not impl"; + }, + saveScreenshotSync: (h: Header, screenshot: string, icon: string): void => { + throw "not impl"; + }, + saveAssetSync: (id: string, filename: string, data: Uint8Array): void => { + throw "not impl"; + }, + listAssetsSync: (id: string): Asset[] => { + throw "not impl"; + }, + // async + getAsync: (h: Header): Promise => { + throw "not impl"; + }, + setAsync: (h: Header, prevVer: any, text?: ScriptText): Promise => { + throw "not impl"; + }, + deleteAsync: (h: Header, prevVer: any): Promise => { + throw "not impl"; + }, + listAsync: (): Promise => { + throw "not impl"; + }, + resetAsync: (): Promise => { + throw "not impl"; + }, + } + return provider; +} \ No newline at end of file From c0fd88f251fc868e99d03ef1ed64468720a9655e Mon Sep 17 00:00:00 2001 From: darzu Date: Tue, 8 Dec 2020 19:36:43 -0800 Subject: [PATCH 19/52] trying out joint workspace --- webapp/src/app.tsx | 1 + webapp/src/scriptmanager.tsx | 5 ++++- webapp/src/workspaces/workspace.ts | 19 +++++++++++++++---- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/webapp/src/app.tsx b/webapp/src/app.tsx index ea0285c141f..9a263c07bca 100644 --- a/webapp/src/app.tsx +++ b/webapp/src/app.tsx @@ -1336,6 +1336,7 @@ export class ProjectView // If user is signed in, sync this project to the cloud. if (this.hasCloudSync()) { + // TODO @darzu: this might not be where we want to attach the user to the project h.cloudUserId = this.getUser()?.id; } diff --git a/webapp/src/scriptmanager.tsx b/webapp/src/scriptmanager.tsx index 11d39cc0eb7..fbe5f35b8d2 100644 --- a/webapp/src/scriptmanager.tsx +++ b/webapp/src/scriptmanager.tsx @@ -185,7 +185,10 @@ export class ScriptManagerDialog extends data.Component workspace.duplicateAsync(header, text, res)) .then(clonedHeader => { // If we're cloud synced, update the cloudSync flag - if (this.props.parent.hasCloudSync()) clonedHeader.cloudUserId = auth.user()?.id; + if (this.props.parent.hasCloudSync()) { + // TODO @darzu: revisit this + clonedHeader.cloudUserId = auth.user()?.id; + } delete clonedHeader.blobId_ delete clonedHeader.blobVersion_ diff --git a/webapp/src/workspaces/workspace.ts b/webapp/src/workspaces/workspace.ts index 395723b4400..8a8cef7d87f 100644 --- a/webapp/src/workspaces/workspace.ts +++ b/webapp/src/workspaces/workspace.ts @@ -18,6 +18,7 @@ import * as cloudWorkspace from "./cloudworkspace" import U = pxt.Util; import Cloud = pxt.Cloud; +import { createJointWorkspace } from "./jointworkspace"; // Avoid importing entire crypto-js /* tslint:disable:no-submodule-imports */ @@ -91,7 +92,12 @@ export function setupWorkspace(kind: WorkspaceKind): void { // TODO @darzu: console.log(`choosing workspace: ${kind}`); implType = kind ?? "browser"; - impl = chooseWorkspace(implType); + const choice = chooseWorkspace(implType); + // TODO @darzu: + if (auth.loggedInSync()) + impl = createJointWorkspace(cloudWorkspace.provider, choice) + else + impl = choice } // TODO @darzu: needed? @@ -143,13 +149,18 @@ async function switchToMemoryWorkspace(reason: string): Promise { } export function getHeaders(withDeleted = false) { + // TODO @darzu: include other stuff... + // return await impl.listAsync(); + // TODO @darzu: we need to consolidate this to one Workspace impl maybeSyncHeadersAsync().done(); const cloudUserId = auth.user()?.id; let r = allScripts.map(e => e.header).filter(h => - (withDeleted || !h.isDeleted) && - !h.isBackup && - (!h.cloudUserId || h.cloudUserId === cloudUserId)) + (withDeleted || !h.isDeleted) + && !h.isBackup + // TODO @darzu: + // && (!h.cloudUserId || h.cloudUserId === cloudUserId) + ) r.sort((a, b) => b.recentUse - a.recentUse) return r } From 6d05fa53f62cf3efb98b1232e74fc608d6d301d6 Mon Sep 17 00:00:00 2001 From: darzu Date: Tue, 8 Dec 2020 20:00:38 -0800 Subject: [PATCH 20/52] experimenting --- pxteditor/workspace.ts | 4 ++ webapp/src/workspaces/realtimeworkspace.ts | 72 ++++++++++++++++++++ webapp/src/workspaces/syncworkspace.ts | 79 ---------------------- 3 files changed, 76 insertions(+), 79 deletions(-) create mode 100644 webapp/src/workspaces/realtimeworkspace.ts delete mode 100644 webapp/src/workspaces/syncworkspace.ts diff --git a/pxteditor/workspace.ts b/pxteditor/workspace.ts index ad62a9ce958..b3078cf356e 100644 --- a/pxteditor/workspace.ts +++ b/pxteditor/workspace.ts @@ -3,6 +3,8 @@ namespace pxt.workspace { export type ScriptText = pxt.Map; + + // TODO @darzu: ugh. why is there a "Project" that is different from a "File". They are nearly identical... export interface Project { header?: Header; text?: ScriptText; @@ -14,6 +16,8 @@ namespace pxt.workspace { url: string; } + // TODO @darzu: why can version be "any" ? that's really annoying to reason about + // TODO @darzu: _rev is a string; modificationTime is an int export type Version = any; export interface File { diff --git a/webapp/src/workspaces/realtimeworkspace.ts b/webapp/src/workspaces/realtimeworkspace.ts new file mode 100644 index 00000000000..181e1a51c74 --- /dev/null +++ b/webapp/src/workspaces/realtimeworkspace.ts @@ -0,0 +1,72 @@ +type Header = pxt.workspace.Header; +type ScriptText = pxt.workspace.ScriptText; +type File = pxt.workspace.File; +type Project = pxt.workspace.File; +type WorkspaceProvider = pxt.workspace.WorkspaceProvider; +type Version = pxt.workspace.Version; +type Asset = pxt.workspace.Version; + +// TODO @darzu: is this the abstraction we want? +// TODO @darzu: replace memory workspace? +interface SyncWorkspaceProvider { + listSync(): Header[]; + getSync(h: Header): File; + setSync(h: Header, prevVersion: Version, text?: ScriptText): Version; + deleteSync?: (h: Header, prevVersion: Version) => void; + resetSync(): void; +} + +export interface RealtimeWorkspaceProvider extends WorkspaceProvider, SyncWorkspaceProvider { } + +export function createRealtimeWorkspace() { + // TODO @darzu: debug logging + console.log(`createRealtimeWorkspace`); + + // TODO @darzu: Project or File ?? + const projects: { [key: string]: File } = {} + + // TODO @darzu: useful? + + const syncProv: SyncWorkspaceProvider = { + listSync: (): Header[] => { + return Object.keys(projects).map(k => projects[k].header); + }, + getSync: (h: Header): File => { + return projects[h.id]; + }, + setSync: (h: Header, prevVersion: Version, text?: ScriptText): Version => { + // TODO @darzu: don't do this if text is null? that's what memoryworkspace does... but db.Table workspace doesn't? + projects[h.id] = { + header: h, + text: text, + // TODO @darzu: version??? + version: prevVersion + "*" + }; + }, + deleteSync: (h: Header, prevVersion: Version): void => { + delete projects[h.id]; + }, + resetSync: (): void => { + Object.keys(projects).forEach(k => delete projects[k]) + }, + } + const provider: RealtimeWorkspaceProvider = { + ...syncProv, + getAsync: (h: Header): Promise => { + return Promise.resolve(syncProv.getSync(h)) + }, + setAsync: (h: Header, prevVer: any, text?: ScriptText): Promise => { + return Promise.resolve(syncProv.setSync(h, prevVer, text)) + }, + deleteAsync: (h: Header, prevVer: any): Promise => { + return Promise.resolve(syncProv.deleteSync(h, prevVer)) + }, + listAsync: (): Promise => { + return Promise.resolve(syncProv.listSync()) + }, + resetAsync: (): Promise => { + return Promise.resolve(syncProv.resetSync()) + }, + } + return provider; +} \ No newline at end of file diff --git a/webapp/src/workspaces/syncworkspace.ts b/webapp/src/workspaces/syncworkspace.ts deleted file mode 100644 index a3fad9df936..00000000000 --- a/webapp/src/workspaces/syncworkspace.ts +++ /dev/null @@ -1,79 +0,0 @@ - - -type Header = pxt.workspace.Header; -type ScriptText = pxt.workspace.ScriptText; -type File = pxt.workspace.File; -type WorkspaceProvider = pxt.workspace.WorkspaceProvider; -type Version = pxt.workspace.Version; -type Asset = pxt.workspace.Version; - -export interface SyncWorkspaceProvider extends WorkspaceProvider { - listSync(): Header[]; - getSync(h: Header): File; - setSync(h: Header, prevVersion: Version, text?: ScriptText): Version; - deleteSync?: (h: Header, prevVersion: Version) => void; - resetSync(): void; - loadedSync?: () => void; - getSyncState?: () => pxt.editor.EditorSyncState; - saveScreenshotSync?: (h: Header, screenshot: string, icon: string) => void; - saveAssetSync?: (id: string, filename: string, data: Uint8Array) => void; - listAssetsSync?: (id: string) => Asset[]; -} - -export function createCachedWorkspace(source: WorkspaceProvider) { - // TODO @darzu: debug logging - console.log(`createCachedWorkspace`); - - // TODO @darzu: useful? - - const provider: SyncWorkspaceProvider = { - // sync - listSync: (): Header[] => { - throw "not impl"; - }, - getSync: (h: Header): File => { - throw "not impl"; - }, - setSync: (h: Header, prevVersion: Version, text?: ScriptText): Version => { - throw "not impl"; - }, - deleteSync: (h: Header, prevVersion: Version): void => { - throw "not impl"; - }, - resetSync: (): void => { - throw "not impl"; - }, - loadedSync: (): void => { - throw "not impl"; - }, - getSyncState: (): pxt.editor.EditorSyncState => { - throw "not impl"; - }, - saveScreenshotSync: (h: Header, screenshot: string, icon: string): void => { - throw "not impl"; - }, - saveAssetSync: (id: string, filename: string, data: Uint8Array): void => { - throw "not impl"; - }, - listAssetsSync: (id: string): Asset[] => { - throw "not impl"; - }, - // async - getAsync: (h: Header): Promise => { - throw "not impl"; - }, - setAsync: (h: Header, prevVer: any, text?: ScriptText): Promise => { - throw "not impl"; - }, - deleteAsync: (h: Header, prevVer: any): Promise => { - throw "not impl"; - }, - listAsync: (): Promise => { - throw "not impl"; - }, - resetAsync: (): Promise => { - throw "not impl"; - }, - } - return provider; -} \ No newline at end of file From 98d30cb4d551b82b82249ddb18dbc1071fc11242 Mon Sep 17 00:00:00 2001 From: darzu Date: Wed, 9 Dec 2020 13:40:47 -0800 Subject: [PATCH 21/52] unreachable helper --- pxtlib/util.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pxtlib/util.ts b/pxtlib/util.ts index ddc7ddc674f..00afd8aa7f4 100644 --- a/pxtlib/util.ts +++ b/pxtlib/util.ts @@ -1409,6 +1409,10 @@ namespace ts.pxtc.Util { return res }) } + + export function unreachable(...ns: never[]): never { + throw "Type error: this code should be unreachable"; + } } namespace ts.pxtc.BrowserImpl { From c7bed33a70551011def201faed26baca176fe6db Mon Sep 17 00:00:00 2001 From: darzu Date: Wed, 9 Dec 2020 13:40:57 -0800 Subject: [PATCH 22/52] header comments --- localtypings/projectheader.d.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/localtypings/projectheader.d.ts b/localtypings/projectheader.d.ts index bd990bf856b..3be70ae37e7 100644 --- a/localtypings/projectheader.d.ts +++ b/localtypings/projectheader.d.ts @@ -35,8 +35,8 @@ declare namespace pxt.workspace { export interface Header extends InstallHeader { id: string; // guid (generated by us) path?: string; // for workspaces that require it - recentUse: number; // seconds since epoch - modificationTime: number; // seconds since epoch + recentUse: number; // seconds since epoch UTC (cloud safe) + modificationTime: number; // seconds since epoch UTC (cloud safe) icon?: string; // icon uri isDeleted: boolean; // mark whether or not a header has been deleted From ffdcbadaa3f65378d1f1fe442c8051b95a7d714d Mon Sep 17 00:00:00 2001 From: darzu Date: Wed, 9 Dec 2020 13:41:31 -0800 Subject: [PATCH 23/52] debug logging --- webapp/src/cloud.ts | 3 ++- webapp/src/workspaces/browserdbworkspace.ts | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/webapp/src/cloud.ts b/webapp/src/cloud.ts index 9e0ce175dab..2697c79520b 100644 --- a/webapp/src/cloud.ts +++ b/webapp/src/cloud.ts @@ -26,8 +26,8 @@ type CloudProject = { export async function listAsync(): Promise { return new Promise(async (resolve, reject) => { // TODO @darzu: this is causing errors? - console.log("cloud.ts:listAsync"); const result = await auth.apiAsync("/api/user/project"); + console.log("cloud.ts:listAsync"); // TODO @darzu: if (result.success) { const userId = auth.user()?.id; const headers = result.resp.map(proj => { @@ -37,6 +37,7 @@ export async function listAsync(): Promise { header.cloudCurrent = true; return header; }); + console.dir(headers) // TODO @darzu: resolve(headers); } else { reject(new Error(result.errmsg)); diff --git a/webapp/src/workspaces/browserdbworkspace.ts b/webapp/src/workspaces/browserdbworkspace.ts index 80f645bf253..79425cf4732 100644 --- a/webapp/src/workspaces/browserdbworkspace.ts +++ b/webapp/src/workspaces/browserdbworkspace.ts @@ -19,10 +19,10 @@ export function createBrowserDbWorkspace(namespace: string): BrowserDbWorkspaceP const textDb = new db.Table(`${prefix}text`); // TODO @darzu: debug logging - console.log(`createBrowserDbWorkspace: ${prefix}`); (async () => { const hdrs: pxt.workspace.Header[] = await headerDb.getAllAsync(); const txts: TextDbEntry[] = await textDb.getAllAsync(); + console.log(`createBrowserDbWorkspace: ${prefix}:`); console.dir(hdrs) console.dir(txts) })(); @@ -30,7 +30,7 @@ export function createBrowserDbWorkspace(namespace: string): BrowserDbWorkspaceP async function listAsync(): Promise { const hdrs: pxt.workspace.Header[] = await headerDb.getAllAsync() // TODO @darzu: debug logging - console.log("browser db headers:") + console.log(`browser db headers ${prefix}:`) console.dir(hdrs.map(h => h.id)) return hdrs } @@ -45,8 +45,8 @@ export function createBrowserDbWorkspace(namespace: string): BrowserDbWorkspaceP async function setAsync(h: Header, prevVer: any, text?: ScriptText): Promise { // TODO @darzu: debug logging if (!text) { - console.log("setAsync without text!") - console.dir(h) + console.log("setAsync without text :(") + // console.dir(h) } else { console.log("setAsync with text :)") } From 0300ad12902c3c13248fececd0ff77d1f330b320 Mon Sep 17 00:00:00 2001 From: darzu Date: Wed, 9 Dec 2020 13:41:58 -0800 Subject: [PATCH 24/52] rename --- .../{realtimeworkspace.ts => memworkspace.ts} | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) rename webapp/src/workspaces/{realtimeworkspace.ts => memworkspace.ts} (88%) diff --git a/webapp/src/workspaces/realtimeworkspace.ts b/webapp/src/workspaces/memworkspace.ts similarity index 88% rename from webapp/src/workspaces/realtimeworkspace.ts rename to webapp/src/workspaces/memworkspace.ts index 181e1a51c74..bc67260da07 100644 --- a/webapp/src/workspaces/realtimeworkspace.ts +++ b/webapp/src/workspaces/memworkspace.ts @@ -8,7 +8,7 @@ type Asset = pxt.workspace.Version; // TODO @darzu: is this the abstraction we want? // TODO @darzu: replace memory workspace? -interface SyncWorkspaceProvider { +export interface SyncWorkspaceProvider { listSync(): Header[]; getSync(h: Header): File; setSync(h: Header, prevVersion: Version, text?: ScriptText): Version; @@ -16,11 +16,12 @@ interface SyncWorkspaceProvider { resetSync(): void; } -export interface RealtimeWorkspaceProvider extends WorkspaceProvider, SyncWorkspaceProvider { } +export interface MemWorkspaceProvider extends WorkspaceProvider, SyncWorkspaceProvider { } -export function createRealtimeWorkspace() { +// TODO @darzu: de-duplicate with memoryworkspace +export function createMemWorkspace() { // TODO @darzu: debug logging - console.log(`createRealtimeWorkspace`); + console.log(`MemWorkspaceProvider`); // TODO @darzu: Project or File ?? const projects: { [key: string]: File } = {} @@ -50,7 +51,7 @@ export function createRealtimeWorkspace() { Object.keys(projects).forEach(k => delete projects[k]) }, } - const provider: RealtimeWorkspaceProvider = { + const provider: MemWorkspaceProvider = { ...syncProv, getAsync: (h: Header): Promise => { return Promise.resolve(syncProv.getSync(h)) From 7da5f13eab2e142b58272efab8716d2cc69a2f64 Mon Sep 17 00:00:00 2001 From: darzu Date: Wed, 9 Dec 2020 13:42:25 -0800 Subject: [PATCH 25/52] taking a stab at synchronization --- .../src/workspaces/synchronizedworkspace.ts | 37 ++++++ webapp/src/workspaces/workspace.ts | 40 +++++- webapp/src/workspaces/workspacebehavior.ts | 115 ++++++++++++++++++ 3 files changed, 187 insertions(+), 5 deletions(-) create mode 100644 webapp/src/workspaces/synchronizedworkspace.ts create mode 100644 webapp/src/workspaces/workspacebehavior.ts diff --git a/webapp/src/workspaces/synchronizedworkspace.ts b/webapp/src/workspaces/synchronizedworkspace.ts new file mode 100644 index 00000000000..3161d54db16 --- /dev/null +++ b/webapp/src/workspaces/synchronizedworkspace.ts @@ -0,0 +1,37 @@ +import { ConflictStrategy, DisjointSetsStrategy, Strategy, synchronize } from "./workspacebehavior"; + +type Header = pxt.workspace.Header; +type ScriptText = pxt.workspace.ScriptText; +type File = pxt.workspace.File; +type WorkspaceProvider = pxt.workspace.WorkspaceProvider; + + +export interface Synchronizable { + syncAsync(): Promise +} + +export function createSynchronizedWorkspace(primary: WorkspaceProvider, cache: T, strat: Strategy): T & Synchronizable { + async function syncAsync() { + // TODO @darzu: parameterize strategy? + await synchronize(primary, cache, strat) + } + + return { + ...cache, + // mutative operations should be kicked off for both + setAsync: async (h, prevVersion, text) => { + // TODO @darzu: don't push to both when disjoint sets strat isn't synchronize + const a = primary.setAsync(h, prevVersion, text) + const b = cache.setAsync(h, prevVersion, text) + await Promise.all([a,b]) + return await a; + }, + deleteAsync: async (h, prevVersion) => { + const a = primary.deleteAsync(h, prevVersion) + const b = cache.deleteAsync(h, prevVersion) + await Promise.all([a,b]) + return await a; + }, + syncAsync, + }; +} diff --git a/webapp/src/workspaces/workspace.ts b/webapp/src/workspaces/workspace.ts index 8a8cef7d87f..e702d30d9f9 100644 --- a/webapp/src/workspaces/workspace.ts +++ b/webapp/src/workspaces/workspace.ts @@ -19,6 +19,10 @@ import * as cloudWorkspace from "./cloudworkspace" import U = pxt.Util; import Cloud = pxt.Cloud; import { createJointWorkspace } from "./jointworkspace"; +import { createBrowserDbWorkspace } from "./browserdbworkspace"; +import { createSynchronizedWorkspace, Synchronizable } from "./synchronizedworkspace"; +import { ConflictStrategy, DisjointSetsStrategy } from "./workspacebehavior"; +import { createMemWorkspace, SyncWorkspaceProvider } from "./memworkspace"; // Avoid importing entire crypto-js /* tslint:disable:no-submodule-imports */ @@ -35,6 +39,7 @@ let allScripts: File[] = []; let headerQ = new U.PromiseQueue(); let impl: WorkspaceProvider; +let implCache: SyncWorkspaceProvider & Synchronizable; let implType: WorkspaceKind; function lookup(id: string) { @@ -92,13 +97,38 @@ export function setupWorkspace(kind: WorkspaceKind): void { // TODO @darzu: console.log(`choosing workspace: ${kind}`); implType = kind ?? "browser"; - const choice = chooseWorkspace(implType); + const localUserChoice = chooseWorkspace(implType); // TODO @darzu: - if (auth.loggedInSync()) - impl = createJointWorkspace(cloudWorkspace.provider, choice) + if (auth.loggedInSync()) { + const localCloud = createBrowserDbWorkspace("cloud-local"); + const cachedCloud = createSynchronizedWorkspace(cloudWorkspace.provider, localCloud, { + conflict: ConflictStrategy.LastWriteWins, + disjointSets: DisjointSetsStrategy.Synchronize + }); + const msPerMin = 1000 * 60 + setInterval(() => { + // TODO @darzu: improve this + console.log("synchronizing with the cloud..."); + cachedCloud.syncAsync() + }, 5 * msPerMin) + + // TODO @darzu: when synchronization causes changes + // data.invalidate("header:*"); + // data.invalidate("text:*"); + + // TODO @darzu: we are assuming these workspaces don't overlapp... + impl = createJointWorkspace(cachedCloud, localUserChoice) + } else - impl = choice -} + impl = localUserChoice + + // TODO @darzu: remove allHeaders etc.. + implCache = createSynchronizedWorkspace(impl, createMemWorkspace(), { + conflict: ConflictStrategy.LastWriteWins, + disjointSets: DisjointSetsStrategy.Synchronize + }); + implCache.syncAsync(); +} // TODO @darzu: needed? export function switchToCloudWorkspace(): string { diff --git a/webapp/src/workspaces/workspacebehavior.ts b/webapp/src/workspaces/workspacebehavior.ts new file mode 100644 index 00000000000..ad3aaf66f1c --- /dev/null +++ b/webapp/src/workspaces/workspacebehavior.ts @@ -0,0 +1,115 @@ +import U = pxt.Util; + +type WorkspaceProvider = pxt.workspace.WorkspaceProvider; +type Header = pxt.workspace.Header; +type Version = pxt.workspace.Version; + +// TODO @darzu: what I need from a header: modificationTime, isDeleted +// TODO @darzu: example: +const exampleRealHeader: Header & {_id: string, _rev: String} = { + "name": "c2", + "meta": {}, + "editor": "blocksprj", + "pubId": "", + "pubCurrent": false, + "target": "arcade", + "targetVersion": "1.3.17", + "cloudUserId": "3341c114-06d5-4ca5-9c2b-b9bb4fb13e81", + "id": "3a30f274-9612-4184-d9ad-e14c99cf81e7", + "recentUse": 1607395785, + "modificationTime": 1607395785, + "path": "c2", + "blobCurrent_": false, + "saveId": null, + "githubCurrent": false, + "cloudCurrent": true, + "cloudVersion": "\"4400a5b3-0000-0100-0000-5fcee9bd0000\"", + "_id": "header--3a30f274-9612-4184-d9ad-e14c99cf81e7", + "_rev": "12-b259964d5d245a44f7141b7c5c41ca23", // TODO @darzu: gotta figure out _rev and _id ... + // TODO @darzu: these are missing in the real header!! + isDeleted: false, + blobId_: null, + blobVersion_: null, +}; + +export enum ConflictStrategy { + LastWriteWins +} +export enum DisjointSetsStrategy { + Synchronize, + DontSynchronize +} + +export interface Strategy { + conflict: ConflictStrategy, + disjointSets: DisjointSetsStrategy +} + +function resolveConflict(a: Header, b: Header, strat: ConflictStrategy): Header { + if (strat === ConflictStrategy.LastWriteWins) + return a.modificationTime > b.modificationTime ? a : b; + U.unreachable(strat); +} + +function hasChanged(a: Header, b: Header): boolean { + // TODO @darzu: use version uuid instead? + return a.modificationTime !== b.modificationTime +} + +async function transfer(h: Header, fromWs: WorkspaceProvider, toWs: WorkspaceProvider) { + const fromPrj = await fromWs.getAsync(h) + const prevVersion: Version = null // TODO @darzu: what do we do with this version thing... + const toRes: Version = await toWs.setAsync(h, prevVersion, fromPrj.text) +} + +export async function synchronize(left: WorkspaceProvider, right: WorkspaceProvider, strat: Strategy) { + // TODO @darzu: + // idea: never delete, only say "isDeleted" is true; can optimize away later + /* + sync scenarios: + cloud & cloud cache (last write wins; + any workspace & memory workspace (memory leads) + + synchronization strategies: + local & remote cloud + local wins? cloud wins? + last write wins? UTC timestamp? + primary & secondary + primary always truth ? + */ + + const lHdrsList = await left.listAsync() + const rHdrsList = await left.listAsync() + const lHdrs = U.toDictionary(lHdrsList, h => h.id) + const rHdrs = U.toDictionary(rHdrsList, h => h.id) + const allHdrsList = [...lHdrsList, ...rHdrsList] + + // determine left-only, overlap, and right-only sets + const overlap = allHdrsList.reduce( + (p: {[key: string]: Header}, n) => lHdrs[n.id] && rHdrs[n.id] ? (p[n.id] = n) && p : p, {}) + const lOnly = allHdrsList.reduce( + (p: {[key: string]: Header}, n) => lHdrs[n.id] && !rHdrs[n.id] ? (p[n.id] = n) && p : p, {}) + const rOnly = allHdrsList.reduce( + (p: {[key: string]: Header}, n) => !lHdrs[n.id] && rHdrs[n.id] ? (p[n.id] = n) && p : p, {}) + + // resolve conflicts + const conflictResults = U.values(overlap).map(h => resolveConflict(lHdrs[h.id], rHdrs[h.id], strat.conflict)) + + // update left + const lChanges = conflictResults.reduce((p: Header[], n) => hasChanged(n, lHdrs[n.id]) ? [...p, n] : p, []) + let lToPush = lChanges + if (strat.disjointSets === DisjointSetsStrategy.Synchronize) + lToPush = [...lToPush, ...U.values(rOnly)] + const lPushPromises = lToPush.map(h => transfer(h, right, left)) + + // update right + const rChanges = conflictResults.reduce((p: Header[], n) => hasChanged(n, rHdrs[n.id]) ? [...p, n] : p, []) + let rToPush = rChanges + if (strat.disjointSets === DisjointSetsStrategy.Synchronize) + lToPush = [...lToPush, ...U.values(lOnly)] + const rPushPromises = rToPush.map(h => transfer(h, left, right)) + + // wait + // TODO @darzu: batching? throttling? incremental? + await Promise.all([...lPushPromises, ...rPushPromises]) +} \ No newline at end of file From dfbd3db33bcfcd362aa824d3bd700c56ca19e8a0 Mon Sep 17 00:00:00 2001 From: darzu Date: Thu, 10 Dec 2020 10:53:49 -0800 Subject: [PATCH 26/52] wip on sync --- .../src/workspaces/synchronizedworkspace.ts | 18 ++++++++- webapp/src/workspaces/workspace.ts | 36 +++++++++++------ webapp/src/workspaces/workspacebehavior.ts | 39 ++++++++++++++++--- 3 files changed, 74 insertions(+), 19 deletions(-) diff --git a/webapp/src/workspaces/synchronizedworkspace.ts b/webapp/src/workspaces/synchronizedworkspace.ts index 3161d54db16..96152da63fb 100644 --- a/webapp/src/workspaces/synchronizedworkspace.ts +++ b/webapp/src/workspaces/synchronizedworkspace.ts @@ -1,3 +1,4 @@ +import { SyncWorkspaceProvider } from "./memworkspace"; import { ConflictStrategy, DisjointSetsStrategy, Strategy, synchronize } from "./workspacebehavior"; type Header = pxt.workspace.Header; @@ -7,17 +8,30 @@ type WorkspaceProvider = pxt.workspace.WorkspaceProvider; export interface Synchronizable { - syncAsync(): Promise + syncAsync(): Promise, + // TODO @darzu: + // lastLeftList(): Header[], + // lastRightList(): Header[], } export function createSynchronizedWorkspace(primary: WorkspaceProvider, cache: T, strat: Strategy): T & Synchronizable { + + // TODO @darzu: debated caching items here + // const lastLeft: Header[] = []; + // const lastRight: Header[] = []; + async function syncAsync() { // TODO @darzu: parameterize strategy? - await synchronize(primary, cache, strat) + const res = await synchronize(primary, cache, strat) + return res.changed } return { ...cache, + // TODO @darzu: + // listAsync: async () => { + + // }, // mutative operations should be kicked off for both setAsync: async (h, prevVersion, text) => { // TODO @darzu: don't push to both when disjoint sets strat isn't synchronize diff --git a/webapp/src/workspaces/workspace.ts b/webapp/src/workspaces/workspace.ts index e702d30d9f9..ce98a56b58d 100644 --- a/webapp/src/workspaces/workspace.ts +++ b/webapp/src/workspaces/workspace.ts @@ -21,7 +21,7 @@ import Cloud = pxt.Cloud; import { createJointWorkspace } from "./jointworkspace"; import { createBrowserDbWorkspace } from "./browserdbworkspace"; import { createSynchronizedWorkspace, Synchronizable } from "./synchronizedworkspace"; -import { ConflictStrategy, DisjointSetsStrategy } from "./workspacebehavior"; +import { ConflictStrategy, DisjointSetsStrategy, wrapInMemCache } from "./workspacebehavior"; import { createMemWorkspace, SyncWorkspaceProvider } from "./memworkspace"; // Avoid importing entire crypto-js @@ -34,6 +34,7 @@ type WorkspaceProvider = pxt.workspace.WorkspaceProvider; type InstallHeader = pxt.workspace.InstallHeader; type File = pxt.workspace.File; +// TODO @darzu: remove. redudant w/ implCache let allScripts: File[] = []; let headerQ = new U.PromiseQueue(); @@ -42,7 +43,10 @@ let impl: WorkspaceProvider; let implCache: SyncWorkspaceProvider & Synchronizable; let implType: WorkspaceKind; -function lookup(id: string) { +function lookup(id: string): File { + // TODO @darzu: + // const hdr = implCache.listSync().find(x => x.header.id == id || x.header.path == id) + // implCache.getSync(); return allScripts.find(x => x.header.id == id || x.header.path == id); } @@ -91,6 +95,7 @@ function chooseWorkspace(kind: WorkspaceKind = "browser"): pxt.workspace.Workspa } } + export function setupWorkspace(kind: WorkspaceKind): void { U.assert(!impl, "workspace set twice"); pxt.log(`workspace: ${kind}`); @@ -105,12 +110,15 @@ export function setupWorkspace(kind: WorkspaceKind): void { conflict: ConflictStrategy.LastWriteWins, disjointSets: DisjointSetsStrategy.Synchronize }); + + // TODO @darzu: improve this const msPerMin = 1000 * 60 - setInterval(() => { - // TODO @darzu: improve this + const doSync = async () => { console.log("synchronizing with the cloud..."); - cachedCloud.syncAsync() - }, 5 * msPerMin) + const changes = await cachedCloud.syncAsync() + console.log(`...changes synced! ${changes}`) + } + setInterval(doSync, 5 * msPerMin) // TODO @darzu: when synchronization causes changes // data.invalidate("header:*"); @@ -123,11 +131,14 @@ export function setupWorkspace(kind: WorkspaceKind): void { impl = localUserChoice // TODO @darzu: remove allHeaders etc.. - implCache = createSynchronizedWorkspace(impl, createMemWorkspace(), { - conflict: ConflictStrategy.LastWriteWins, - disjointSets: DisjointSetsStrategy.Synchronize - }); + implCache = wrapInMemCache(impl) implCache.syncAsync(); + + // TODO @darzu: + // if (changes.length) { + // data.invalidate("header:*"); + // data.invalidate("text:*"); + // } } // TODO @darzu: needed? @@ -185,7 +196,10 @@ export function getHeaders(withDeleted = false) { // TODO @darzu: we need to consolidate this to one Workspace impl maybeSyncHeadersAsync().done(); const cloudUserId = auth.user()?.id; - let r = allScripts.map(e => e.header).filter(h => + // TODO @darzu: use allScripts still? + // let r = allScripts.map(e => e.header) + let r = implCache.listSync() + .filter(h => (withDeleted || !h.isDeleted) && !h.isBackup // TODO @darzu: diff --git a/webapp/src/workspaces/workspacebehavior.ts b/webapp/src/workspaces/workspacebehavior.ts index ad3aaf66f1c..3c48b8c7c02 100644 --- a/webapp/src/workspaces/workspacebehavior.ts +++ b/webapp/src/workspaces/workspacebehavior.ts @@ -1,3 +1,5 @@ +import { createMemWorkspace, SyncWorkspaceProvider } from "./memworkspace"; +import { createSynchronizedWorkspace, Synchronizable } from "./synchronizedworkspace"; import U = pxt.Util; type WorkspaceProvider = pxt.workspace.WorkspaceProvider; @@ -56,14 +58,23 @@ function hasChanged(a: Header, b: Header): boolean { return a.modificationTime !== b.modificationTime } -async function transfer(h: Header, fromWs: WorkspaceProvider, toWs: WorkspaceProvider) { +async function transfer(h: Header, fromWs: WorkspaceProvider, toWs: WorkspaceProvider): Promise
{ const fromPrj = await fromWs.getAsync(h) const prevVersion: Version = null // TODO @darzu: what do we do with this version thing... const toRes: Version = await toWs.setAsync(h, prevVersion, fromPrj.text) + return h; } -export async function synchronize(left: WorkspaceProvider, right: WorkspaceProvider, strat: Strategy) { - // TODO @darzu: +export interface SyncResult { + changed: Header[], + left: Header[], + right: Header[], +} + +export async function synchronize(left: WorkspaceProvider, right: WorkspaceProvider, strat: Strategy): Promise { + // TODO @darzu: add "on changes identified" handler so we can show in-progress syncing + + // TODO @darzu: thoughts & notes // idea: never delete, only say "isDeleted" is true; can optimize away later /* sync scenarios: @@ -79,7 +90,7 @@ export async function synchronize(left: WorkspaceProvider, right: WorkspaceProvi */ const lHdrsList = await left.listAsync() - const rHdrsList = await left.listAsync() + const rHdrsList = await right.listAsync() const lHdrs = U.toDictionary(lHdrsList, h => h.id) const rHdrs = U.toDictionary(rHdrsList, h => h.id) const allHdrsList = [...lHdrsList, ...rHdrsList] @@ -106,10 +117,26 @@ export async function synchronize(left: WorkspaceProvider, right: WorkspaceProvi const rChanges = conflictResults.reduce((p: Header[], n) => hasChanged(n, rHdrs[n.id]) ? [...p, n] : p, []) let rToPush = rChanges if (strat.disjointSets === DisjointSetsStrategy.Synchronize) - lToPush = [...lToPush, ...U.values(lOnly)] + rToPush = [...rToPush, ...U.values(lOnly)] const rPushPromises = rToPush.map(h => transfer(h, left, right)) // wait // TODO @darzu: batching? throttling? incremental? - await Promise.all([...lPushPromises, ...rPushPromises]) + const changed = await Promise.all([...lPushPromises, ...rPushPromises]) + + // return final results + const lRes = [...U.values(lOnly), ...lToPush] + const rRes = [...U.values(rOnly), ...rToPush] + return { + changed, + left: lRes, + right: rRes + } +} + +export function wrapInMemCache(ws: WorkspaceProvider): SyncWorkspaceProvider & WorkspaceProvider & Synchronizable { + return createSynchronizedWorkspace(ws, createMemWorkspace(), { + conflict: ConflictStrategy.LastWriteWins, + disjointSets: DisjointSetsStrategy.Synchronize + }); } \ No newline at end of file From a7523ad78686d819ab6149c1389887e456fb50be Mon Sep 17 00:00:00 2001 From: darzu Date: Fri, 11 Dec 2020 15:56:59 -0800 Subject: [PATCH 27/52] trying cloud sync workspace again --- webapp/src/workspaces/cloudsyncworkspace.ts | 362 ++++++++++++++++++++ webapp/src/workspaces/cloudworkspace.ts | 2 + webapp/src/workspaces/workspace.ts | 2 +- 3 files changed, 365 insertions(+), 1 deletion(-) create mode 100644 webapp/src/workspaces/cloudsyncworkspace.ts diff --git a/webapp/src/workspaces/cloudsyncworkspace.ts b/webapp/src/workspaces/cloudsyncworkspace.ts new file mode 100644 index 00000000000..c041aec9f16 --- /dev/null +++ b/webapp/src/workspaces/cloudsyncworkspace.ts @@ -0,0 +1,362 @@ +import { del } from "request"; +import { ConflictStrategy, DisjointSetsStrategy, Strategy } from "./workspacebehavior"; +import U = pxt.Util; + +type Header = pxt.workspace.Header; +type ScriptText = pxt.workspace.ScriptText; +type F = pxt.workspace.File; +type Version = pxt.workspace.Version; +type WorkspaceProvider = pxt.workspace.WorkspaceProvider; + +// TODO @darzu: +// cache invalidation + +interface CachedWorkspaceProvider extends WorkspaceProvider { + getLastModTime(): number, + updateCache(otherLastModTime?: number): Promise, + lastHeaders(): Header[], + firstSync(): Promise, +} + +function computeLastModTime(hdrs: Header[]): number { + return hdrs.reduce((p, n) => Math.max(n.modificationTime, p), 0) +} +function hasChanged(a: Header, b: Header): boolean { + // TODO @darzu: use version uuid instead? + return a.modificationTime !== b.modificationTime +} + +// TODO @darzu: use cases: multi-tab and cloud +function createCachedWorkspace(ws: WorkspaceProvider): CachedWorkspaceProvider { + let cacheHdrs: Header[] = [] + let cacheHdrsMap: {[id: string]: Header} = {}; + let cacheProjs: {[id: string]: F} = {}; + + let cacheModTime: number = 0; + + function getLastModTime(): number { + return cacheModTime; + } + + let pendingUpdate: Promise = updateCacheInternal(); + const firstUpdate = pendingUpdate; + async function updateCache(otherLastModTime?:number): Promise { + if (pendingUpdate.isPending()) + return pendingUpdate + pendingUpdate = updateCacheInternal(otherLastModTime) + return pendingUpdate + } + + function eraseCache() { + cacheHdrs = [] + cacheHdrsMap = {} + cacheProjs = {} + cacheModTime = 0 + } + + async function updateCacheInternal(otherLastModTime?:number): Promise { + // remember our old cache, we might keep items from it later + const oldHdrs = cacheHdrs + const oldHdrsMap = cacheHdrsMap + const oldProjs = cacheProjs + const oldModTime = cacheModTime + + if (otherLastModTime && otherLastModTime !== cacheModTime) { + // we've been told to invalidate, but we don't know specific + // headers yet so do the conservative thing and reset all + eraseCache() + } + + const newHdrs = await ws.listAsync() + const newLastModTime = computeLastModTime(newHdrs); + if (newLastModTime === oldModTime) { + // no change, keep the old cache + cacheHdrs = oldHdrs + cacheHdrsMap = oldHdrsMap + cacheProjs = oldProjs + return false + } + + // compute header differences and clear old cache entries + const newHdrsMap = U.toDictionary(newHdrs, h => h.id) + const newProjs = oldProjs + for (let id of Object.keys(newProjs)) { + const newHdr = newHdrsMap[id] + if (!newHdr || hasChanged(newHdr, oldHdrsMap[id])) + delete newProjs[id] + } + + // save results + cacheModTime = newLastModTime + cacheProjs = newProjs + cacheHdrs = newHdrs + cacheHdrsMap = newHdrsMap + return true; + } + + async function listAsync(): Promise { + await pendingUpdate; + return cacheHdrs + } + async function getAsync(h: Header): Promise { + await pendingUpdate; + if (!cacheProjs[h.id]) { + // fetch + // TODO @darzu: use / cooperate with worklist? + const proj = await ws.getAsync(h) + cacheProjs[h.id] = proj + } + return cacheProjs[h.id]; + } + async function setAsync(h: Header, prevVer: Version, text?: ScriptText): Promise { + await pendingUpdate; + cacheProjs[h.id] = { + header: h, + text, + version: null // TODO @darzu: + } + cacheModTime = Math.max(cacheModTime, h.modificationTime) + const res = await ws.setAsync(h, prevVer, text) + return res; + } + async function deleteAsync(h: Header, prevVer: Version): Promise { + await pendingUpdate; + delete cacheProjs[h.id]; + // TODO @darzu: how to handle mod time with delete? + // TODO @darzu: we should probably enforce soft delete everywhere... + cacheModTime = Math.max(cacheModTime, h.modificationTime) + const res = await ws.deleteAsync(h, prevVer) + return res; + } + async function resetAsync() { + await pendingUpdate; + eraseCache(); + await ws.resetAsync() + } + + const provider: CachedWorkspaceProvider = { + // cache + getLastModTime, + updateCache, + lastHeaders: () => cacheHdrs, + firstSync: () => firstUpdate, + // workspace + getAsync, + setAsync, + deleteAsync, + listAsync, + resetAsync, + } + + return provider; +} + +interface CloudSyncWorkspace extends WorkspaceProvider { +} + +function createCloudSyncWorkspace(cloud: WorkspaceProvider, cloudLocal: WorkspaceProvider) { + let lastHdrs: Header[] = [] + let lastHdrsMap: {[key: string]: Header}; + let lastProjs: {[key: string]: F}; + let lastSync: number = null; + + async function memCacheInvalidate() { + // TODO @darzu: short circuit if full sync ? + await synchronize(); + } + async function listAsync(): Promise { + if (!lastSync || !lastHdrs.length) + await synchronize(); + return lastHdrs; + } + async function getAsync(h: Header): Promise { + if (!lastSync) + await synchronize(); + if (lastProjs[h.id]) { + // cache hit + // TODO @darzu: this isn't good for multi-tabs maybe ? + return lastProjs[h.id] + } + const res = await cloudLocal.getAsync(h) + lastProjs[h.id] = res + return res; + } + async function setAsync(h: Header, prevVer: any, text?: ScriptText): Promise { + if (!lastSync) + await synchronize(); + lastProjs[h.id] = { + header: h, + version: undefined, // TODO @darzu: + text, + } + const l = cloud.setAsync(h, prevVer, text) + // TODO @darzu: wait for cloud? + const r = cloudLocal.setAsync(h, prevVer, text) + return await r + } + async function deleteAsync(h: Header, prevVer: any): Promise { + if (!lastSync) + await synchronize(); + throw "not impl" + } + async function resetAsync() { + if (!lastSync) + await synchronize(); + throw "not impl" + } + + function resolveConflict(a: Header, b: Header, strat: ConflictStrategy): Header { + if (strat === ConflictStrategy.LastWriteWins) + return a.modificationTime > b.modificationTime ? a : b; + U.unreachable(strat); + } + async function transfer(h: Header, fromWs: WorkspaceProvider, toWs: WorkspaceProvider): Promise
{ + const fromPrj = await fromWs.getAsync(h) + + // TODO @darzu: + lastProjs[h.id] = fromPrj; + + const prevVersion: Version = null // TODO @darzu: what do we do with this version thing... + const toRes: Version = await toWs.setAsync(h, prevVersion, fromPrj.text) + return h; + } + async function synchronize(): Promise { + // TODO @darzu: re-generalize? + const left = cloud; + const right = cloudLocal; + const strat = { + conflict: ConflictStrategy.LastWriteWins, + disjointSets: DisjointSetsStrategy.Synchronize + } + + // TODO @darzu: short circuit if there aren't changes ? + const lHdrsList = await left.listAsync() + const rHdrsList = await right.listAsync() + + const lHdrs = U.toDictionary(lHdrsList, h => h.id) + const rHdrs = U.toDictionary(rHdrsList, h => h.id) + const allHdrsList = [...lHdrsList, ...rHdrsList] + + // determine left-only, overlap, and right-only sets + const overlap = allHdrsList.reduce( + (p: {[key: string]: Header}, n) => lHdrs[n.id] && rHdrs[n.id] ? (p[n.id] = n) && p : p, {}) + const lOnly = allHdrsList.reduce( + (p: {[key: string]: Header}, n) => lHdrs[n.id] && !rHdrs[n.id] ? (p[n.id] = n) && p : p, {}) + const rOnly = allHdrsList.reduce( + (p: {[key: string]: Header}, n) => !lHdrs[n.id] && rHdrs[n.id] ? (p[n.id] = n) && p : p, {}) + + // resolve conflicts + const conflictResults = U.values(overlap).map(h => resolveConflict(lHdrs[h.id], rHdrs[h.id], strat.conflict)) + + // update left + const lChanges = conflictResults.reduce((p: Header[], n) => hasChanged(n, lHdrs[n.id]) ? [...p, n] : p, []) + let lToPush = lChanges + if (strat.disjointSets === DisjointSetsStrategy.Synchronize) + lToPush = [...lToPush, ...U.values(rOnly)] + const lPushPromises = lToPush.map(h => transfer(h, right, left)) + + // update right + const rChanges = conflictResults.reduce((p: Header[], n) => hasChanged(n, rHdrs[n.id]) ? [...p, n] : p, []) + let rToPush = rChanges + if (strat.disjointSets === DisjointSetsStrategy.Synchronize) + rToPush = [...rToPush, ...U.values(lOnly)] + const rPushPromises = rToPush.map(h => transfer(h, left, right)) + + // wait + // TODO @darzu: batching? throttling? incremental? + const changed = await Promise.all([...lPushPromises, ...rPushPromises]) + + // return final results + const lRes = [...U.values(lOnly), ...lToPush] + const rRes = [...U.values(rOnly), ...rToPush] + + // TODO @darzu: + lastHdrs = lRes; + lastHdrsMap = U.toDictionary(lastHdrs, h => h.id) + lastSync = Date.now() + + return; + } + + const provider: CloudSyncWorkspace = { + getAsync, + setAsync, + deleteAsync, + listAsync, + resetAsync, + } + + return provider; +} + +// TODO @darzu: below is the code for multi-tab synchronizing + +// // this key is the max modificationTime value of the allHeaders +// // it is used to track if allHeaders need to be refreshed (syncAsync) +// let sessionID: string = ""; +// export function isHeadersSessionOutdated() { +// return pxt.storage.getLocal('workspacesessionid') != sessionID; +// } +// function maybeSyncHeadersAsync(): Promise { +// if (isHeadersSessionOutdated()) // another tab took control +// return syncAsync().then(() => { }) +// return Promise.resolve(); +// } +// function refreshHeadersSession() { +// // TODO @darzu: carefully handle this +// // use # of scripts + time of last mod as key +// sessionID = allScripts.length + ' ' + allScripts +// .map(h => h.header.modificationTime) +// .reduce((l, r) => Math.max(l, r), 0) +// .toString() +// if (isHeadersSessionOutdated()) { +// pxt.storage.setLocal('workspacesessionid', sessionID); +// pxt.debug(`workspace: refreshed headers session to ${sessionID}`); +// data.invalidate("header:*"); +// data.invalidate("text:*"); +// } +// } +// // this is an identifier for the current frame +// // in order to lock headers for editing +// const workspaceID: string = pxt.Util.guidGen(); +// export function acquireHeaderSession(h: Header) { +// if (h) +// pxt.storage.setLocal('workspaceheadersessionid:' + h.id, workspaceID); +// } +// function clearHeaderSession(h: Header) { +// if (h) +// pxt.storage.removeLocal('workspaceheadersessionid:' + h.id); +// } +// export function isHeaderSessionOutdated(h: Header): boolean { +// if (!h) return false; +// const sid = pxt.storage.getLocal('workspaceheadersessionid:' + h.id); +// return sid && sid != workspaceID; +// } +// function checkHeaderSession(h: Header): void { +// if (isHeaderSessionOutdated(h)) { +// pxt.tickEvent(`workspace.conflict.header`); +// core.errorNotification(lf("This project is already opened elsewhere.")) +// pxt.Util.assert(false, "trying to access outdated session") +// } +// } + +// TODO @darzu: from webapp: +// loadHeaderAsync(h: pxt.workspace.Header, editorState?: pxt.editor.EditorState): Promise { +// if (!h) +// return Promise.resolve() + +// const checkAsync = this.tryCheckTargetVersionAsync(h.targetVersion); +// if (checkAsync) +// return checkAsync.then(() => this.openHome()); + +// let p = Promise.resolve(); +// if (workspace.isHeadersSessionOutdated()) { // reload header before loading +// pxt.log(`sync before load`) +// p = p.then(() => workspace.syncAsync().then(() => { })) +// } +// return p.then(() => { +// workspace.acquireHeaderSession(h); +// if (!h) return Promise.resolve(); +// else return this.internalLoadHeaderAsync(h, editorState); +// }) +// } \ No newline at end of file diff --git a/webapp/src/workspaces/cloudworkspace.ts b/webapp/src/workspaces/cloudworkspace.ts index 8c5914aca6b..cbfe515f092 100644 --- a/webapp/src/workspaces/cloudworkspace.ts +++ b/webapp/src/workspaces/cloudworkspace.ts @@ -10,6 +10,8 @@ export const provider: WorkspaceProvider = { resetAsync: cloud.resetAsync, } +// TODO @darzu: throttled workspace ?? + // TODO @darzu: do we need a subscription here? // export function init() { // data.subscribe(userSubscriber, auth.LOGGED_IN); diff --git a/webapp/src/workspaces/workspace.ts b/webapp/src/workspaces/workspace.ts index ce98a56b58d..3bcda441aaf 100644 --- a/webapp/src/workspaces/workspace.ts +++ b/webapp/src/workspaces/workspace.ts @@ -426,7 +426,7 @@ export function saveAsync(h: Header, text?: ScriptText, isCloud?: boolean): Prom U.assert(h.target == pxt.appTarget.id); if (h.temporary) { - // TODO @darzu: lol... what abstraction does this fit? + // TODO @darzu: sigh. what is "temporary" mean return Promise.resolve() } From bc8a1a394ab9f3463ef0a47e82aa96ed07afd78b Mon Sep 17 00:00:00 2001 From: darzu Date: Sat, 12 Dec 2020 10:57:44 -0800 Subject: [PATCH 28/52] lots of work on cloud sync --- webapp/src/workspaces/cloudsyncworkspace.ts | 160 ++++++++++---------- 1 file changed, 83 insertions(+), 77 deletions(-) diff --git a/webapp/src/workspaces/cloudsyncworkspace.ts b/webapp/src/workspaces/cloudsyncworkspace.ts index c041aec9f16..b7e100417a8 100644 --- a/webapp/src/workspaces/cloudsyncworkspace.ts +++ b/webapp/src/workspaces/cloudsyncworkspace.ts @@ -13,9 +13,12 @@ type WorkspaceProvider = pxt.workspace.WorkspaceProvider; interface CachedWorkspaceProvider extends WorkspaceProvider { getLastModTime(): number, - updateCache(otherLastModTime?: number): Promise, - lastHeaders(): Header[], + synchronize(expectedLastModTime?: number): Promise, + pendingSync(): Promise, firstSync(): Promise, + headersSync(): Header[], + hasSync(h: Header): boolean, + tryGetSync(h: Header): F } function computeLastModTime(hdrs: Header[]): number { @@ -38,12 +41,12 @@ function createCachedWorkspace(ws: WorkspaceProvider): CachedWorkspaceProvider { return cacheModTime; } - let pendingUpdate: Promise = updateCacheInternal(); + let pendingUpdate: Promise = synchronizeInternal(); const firstUpdate = pendingUpdate; - async function updateCache(otherLastModTime?:number): Promise { + async function synchronize(otherLastModTime?:number): Promise { if (pendingUpdate.isPending()) return pendingUpdate - pendingUpdate = updateCacheInternal(otherLastModTime) + pendingUpdate = synchronizeInternal(otherLastModTime) return pendingUpdate } @@ -54,14 +57,14 @@ function createCachedWorkspace(ws: WorkspaceProvider): CachedWorkspaceProvider { cacheModTime = 0 } - async function updateCacheInternal(otherLastModTime?:number): Promise { + async function synchronizeInternal(expectedLastModTime?:number): Promise { // remember our old cache, we might keep items from it later const oldHdrs = cacheHdrs const oldHdrsMap = cacheHdrsMap const oldProjs = cacheProjs const oldModTime = cacheModTime - if (otherLastModTime && otherLastModTime !== cacheModTime) { + if (expectedLastModTime && expectedLastModTime !== cacheModTime) { // we've been told to invalidate, but we don't know specific // headers yet so do the conservative thing and reset all eraseCache() @@ -137,9 +140,12 @@ function createCachedWorkspace(ws: WorkspaceProvider): CachedWorkspaceProvider { const provider: CachedWorkspaceProvider = { // cache getLastModTime, - updateCache, - lastHeaders: () => cacheHdrs, + synchronize, + pendingSync: () => pendingUpdate, firstSync: () => firstUpdate, + headersSync: () => cacheHdrs, + hasSync: h => !!cacheHdrsMap[h.id], + tryGetSync: h => cacheProjs[h.id], // workspace getAsync, setAsync, @@ -151,58 +157,28 @@ function createCachedWorkspace(ws: WorkspaceProvider): CachedWorkspaceProvider { return provider; } -interface CloudSyncWorkspace extends WorkspaceProvider { +interface CloudSyncWorkspace extends WorkspaceProvider, CachedWorkspaceProvider { } function createCloudSyncWorkspace(cloud: WorkspaceProvider, cloudLocal: WorkspaceProvider) { - let lastHdrs: Header[] = [] - let lastHdrsMap: {[key: string]: Header}; - let lastProjs: {[key: string]: F}; - let lastSync: number = null; - - async function memCacheInvalidate() { - // TODO @darzu: short circuit if full sync ? - await synchronize(); - } - async function listAsync(): Promise { - if (!lastSync || !lastHdrs.length) - await synchronize(); - return lastHdrs; - } - async function getAsync(h: Header): Promise { - if (!lastSync) - await synchronize(); - if (lastProjs[h.id]) { - // cache hit - // TODO @darzu: this isn't good for multi-tabs maybe ? - return lastProjs[h.id] - } - const res = await cloudLocal.getAsync(h) - lastProjs[h.id] = res - return res; - } - async function setAsync(h: Header, prevVer: any, text?: ScriptText): Promise { - if (!lastSync) - await synchronize(); - lastProjs[h.id] = { - header: h, - version: undefined, // TODO @darzu: - text, - } - const l = cloud.setAsync(h, prevVer, text) - // TODO @darzu: wait for cloud? - const r = cloudLocal.setAsync(h, prevVer, text) - return await r - } - async function deleteAsync(h: Header, prevVer: any): Promise { - if (!lastSync) - await synchronize(); - throw "not impl" - } - async function resetAsync() { - if (!lastSync) - await synchronize(); - throw "not impl" + const cloudCache = createCachedWorkspace(cloud); + const localCache = createCachedWorkspace(cloudLocal); + + const firstCachePull = Promise.all([cloudCache.firstSync(), localCache.firstSync()]) + const pendingCacheSync = () => Promise.all([cloudCache.pendingSync(), localCache.pendingSync()]) + const getLastModTime = () => Math.max(cloudCache.getLastModTime(), localCache.getLastModTime()) + + // TODO @darzu: multi-tab safety for cloudLocal + + // TODO @darzu: when two workspaces disagree on last mod time, we should sync? + + let pendingSync: Promise = synchronizeInternal(); // TODO @darzu: kick this off immediately? + const firstSync = pendingSync; + async function synchronize(expectedLastModTime?:number): Promise { + if (pendingSync.isPending()) + return pendingSync + pendingSync = synchronizeInternal() + return pendingSync } function resolveConflict(a: Header, b: Header, strat: ConflictStrategy): Header { @@ -213,17 +189,25 @@ function createCloudSyncWorkspace(cloud: WorkspaceProvider, cloudLocal: Workspac async function transfer(h: Header, fromWs: WorkspaceProvider, toWs: WorkspaceProvider): Promise
{ const fromPrj = await fromWs.getAsync(h) - // TODO @darzu: - lastProjs[h.id] = fromPrj; + // TODO @darzu: caches? const prevVersion: Version = null // TODO @darzu: what do we do with this version thing... + // TODO @darzu: track pending saves const toRes: Version = await toWs.setAsync(h, prevVersion, fromPrj.text) return h; } - async function synchronize(): Promise { + async function synchronizeInternal(): Promise { + await pendingCacheSync() + + if (cloudCache.getLastModTime() === localCache.getLastModTime()) { + // we're synced up ? + return false + } + // TODO @darzu: compare mod times to see if we need to sync? + // TODO @darzu: re-generalize? - const left = cloud; - const right = cloudLocal; + const left = cloudCache; + const right = localCache; const strat = { conflict: ConflictStrategy.LastWriteWins, disjointSets: DisjointSetsStrategy.Synchronize @@ -266,24 +250,46 @@ function createCloudSyncWorkspace(cloud: WorkspaceProvider, cloudLocal: Workspac // TODO @darzu: batching? throttling? incremental? const changed = await Promise.all([...lPushPromises, ...rPushPromises]) - // return final results - const lRes = [...U.values(lOnly), ...lToPush] - const rRes = [...U.values(rOnly), ...rToPush] - - // TODO @darzu: - lastHdrs = lRes; - lastHdrsMap = U.toDictionary(lastHdrs, h => h.id) - lastSync = Date.now() + // TODO @darzu: what about mod time changes? + return changed.length >= 0; + } - return; + async function listAsync(): Promise { + await pendingSync + return localCache.listAsync() + } + async function getAsync(h: Header): Promise { + await pendingSync + return localCache.getAsync(h) + } + async function setAsync(h: Header, prevVer: any, text?: ScriptText): Promise { + await pendingSync + throw "not impl" + } + async function deleteAsync(h: Header, prevVer: any): Promise { + await pendingSync + throw "not impl" + } + async function resetAsync() { + await pendingSync + throw "not impl" } const provider: CloudSyncWorkspace = { - getAsync, - setAsync, - deleteAsync, - listAsync, - resetAsync, + // cache + getLastModTime, + synchronize, + pendingSync: () => pendingSync, + firstSync: () => firstSync, + headersSync: () => localCache.headersSync(), + hasSync: h => localCache.hasSync(h), + tryGetSync: h => localCache.tryGetSync(h), + // workspace + getAsync, + setAsync, + deleteAsync, + listAsync, + resetAsync, } return provider; From b8e839368b8d8642377e948fbff5fed1207d78aa Mon Sep 17 00:00:00 2001 From: darzu Date: Sun, 13 Dec 2020 14:38:40 -0800 Subject: [PATCH 29/52] much better joint workspace --- webapp/src/workspaces/cloudsyncworkspace.ts | 32 ++++-- webapp/src/workspaces/jointworkspace.ts | 109 +++++++++++++++++--- webapp/src/workspaces/workspace.ts | 46 ++++++--- webapp/src/workspaces/workspacebehavior.ts | 5 + 4 files changed, 151 insertions(+), 41 deletions(-) diff --git a/webapp/src/workspaces/cloudsyncworkspace.ts b/webapp/src/workspaces/cloudsyncworkspace.ts index b7e100417a8..e25419e93d2 100644 --- a/webapp/src/workspaces/cloudsyncworkspace.ts +++ b/webapp/src/workspaces/cloudsyncworkspace.ts @@ -11,12 +11,12 @@ type WorkspaceProvider = pxt.workspace.WorkspaceProvider; // TODO @darzu: // cache invalidation -interface CachedWorkspaceProvider extends WorkspaceProvider { +export interface CachedWorkspaceProvider extends WorkspaceProvider { getLastModTime(): number, synchronize(expectedLastModTime?: number): Promise, pendingSync(): Promise, firstSync(): Promise, - headersSync(): Header[], + listSync(): Header[], hasSync(h: Header): boolean, tryGetSync(h: Header): F } @@ -30,7 +30,7 @@ function hasChanged(a: Header, b: Header): boolean { } // TODO @darzu: use cases: multi-tab and cloud -function createCachedWorkspace(ws: WorkspaceProvider): CachedWorkspaceProvider { +export function createCachedWorkspace(ws: WorkspaceProvider): CachedWorkspaceProvider { let cacheHdrs: Header[] = [] let cacheHdrsMap: {[id: string]: Header} = {}; let cacheProjs: {[id: string]: F} = {}; @@ -41,6 +41,7 @@ function createCachedWorkspace(ws: WorkspaceProvider): CachedWorkspaceProvider { return cacheModTime; } + // TODO @darzu: do we want to kick off the first sync at construction? Side-effects at construction are usually bad.. let pendingUpdate: Promise = synchronizeInternal(); const firstUpdate = pendingUpdate; async function synchronize(otherLastModTime?:number): Promise { @@ -143,7 +144,7 @@ function createCachedWorkspace(ws: WorkspaceProvider): CachedWorkspaceProvider { synchronize, pendingSync: () => pendingUpdate, firstSync: () => firstUpdate, - headersSync: () => cacheHdrs, + listSync: () => cacheHdrs, hasSync: h => !!cacheHdrsMap[h.id], tryGetSync: h => cacheProjs[h.id], // workspace @@ -157,10 +158,10 @@ function createCachedWorkspace(ws: WorkspaceProvider): CachedWorkspaceProvider { return provider; } -interface CloudSyncWorkspace extends WorkspaceProvider, CachedWorkspaceProvider { +export interface CloudSyncWorkspace extends CachedWorkspaceProvider { } -function createCloudSyncWorkspace(cloud: WorkspaceProvider, cloudLocal: WorkspaceProvider) { +export function createCloudSyncWorkspace(cloud: WorkspaceProvider, cloudLocal: WorkspaceProvider): CloudSyncWorkspace { const cloudCache = createCachedWorkspace(cloud); const localCache = createCachedWorkspace(cloudLocal); @@ -169,7 +170,6 @@ function createCloudSyncWorkspace(cloud: WorkspaceProvider, cloudLocal: Workspac const getLastModTime = () => Math.max(cloudCache.getLastModTime(), localCache.getLastModTime()) // TODO @darzu: multi-tab safety for cloudLocal - // TODO @darzu: when two workspaces disagree on last mod time, we should sync? let pendingSync: Promise = synchronizeInternal(); // TODO @darzu: kick this off immediately? @@ -213,6 +213,9 @@ function createCloudSyncWorkspace(cloud: WorkspaceProvider, cloudLocal: Workspac disjointSets: DisjointSetsStrategy.Synchronize } + // wait for each side to sync + await Promise.all([cloudCache.synchronize(), localCache.synchronize()]) + // TODO @darzu: short circuit if there aren't changes ? const lHdrsList = await left.listAsync() const rHdrsList = await right.listAsync() @@ -264,15 +267,22 @@ function createCloudSyncWorkspace(cloud: WorkspaceProvider, cloudLocal: Workspac } async function setAsync(h: Header, prevVer: any, text?: ScriptText): Promise { await pendingSync - throw "not impl" + // TODO @darzu: use a queue to sync to backend + const cloudPromise = cloudCache.setAsync(h, prevVer, text) + // TODO @darzu: also what to do with the return value ? + return await localCache.setAsync(h, prevVer, text) } async function deleteAsync(h: Header, prevVer: any): Promise { await pendingSync - throw "not impl" + // TODO @darzu: use a queue to sync to backend + const cloudPromise = cloudCache.deleteAsync(h, prevVer) + await localCache.deleteAsync(h, prevVer) } async function resetAsync() { await pendingSync - throw "not impl" + // TODO @darzu: do we really want to reset the cloud ever? + // await Promise.all([cloudCache.resetAsync(), localCache.resetAsync()]) + return Promise.resolve(); } const provider: CloudSyncWorkspace = { @@ -281,7 +291,7 @@ function createCloudSyncWorkspace(cloud: WorkspaceProvider, cloudLocal: Workspac synchronize, pendingSync: () => pendingSync, firstSync: () => firstSync, - headersSync: () => localCache.headersSync(), + listSync: () => localCache.listSync(), hasSync: h => localCache.hasSync(h), tryGetSync: h => localCache.tryGetSync(h), // workspace diff --git a/webapp/src/workspaces/jointworkspace.ts b/webapp/src/workspaces/jointworkspace.ts index 061d3a5fde1..ebbd589a5a4 100644 --- a/webapp/src/workspaces/jointworkspace.ts +++ b/webapp/src/workspaces/jointworkspace.ts @@ -1,26 +1,33 @@ +import { CachedWorkspaceProvider } from "./cloudsyncworkspace"; + type Header = pxt.workspace.Header; type ScriptText = pxt.workspace.ScriptText; type File = pxt.workspace.File; type WorkspaceProvider = pxt.workspace.WorkspaceProvider; -export function createJointWorkspace(primary: WorkspaceProvider, ...others: WorkspaceProvider[]): WorkspaceProvider { +async function unique(...listFns: (() => Promise)[]) { + const allHdrs = (await Promise.all(listFns.map(ls => ls()))) + .reduce((p, n) => [...p, ...n], []) + const seenHdrs: { [key: string]: boolean } = {} + // de-duplicate headers (prefering earlier ones) + const res = allHdrs.reduce((p, n) => { + if (seenHdrs[n.id]) + return p; + seenHdrs[n.id] = true; + return [...p, n] + }, []) + return res; +} + +// TODO @darzu: still useful? else cull +export function createJointWorkspace2(primary: WorkspaceProvider, ...others: WorkspaceProvider[]): WorkspaceProvider { const all: WorkspaceProvider[] = [primary, ...others]; // TODO @darzu: debug logging - console.log(`createJointWorkspace`); + console.log(`createJointWorkspace2`); async function listAsync(): Promise { - const allHdrs = (await Promise.all(all.map(ws => ws.listAsync()))) - .reduce((p, n) => [...p, ...n], []) - const seenHdrs: { [key: string]: boolean } = {} - // de-duplicate headers (prefering earlier ones) - const res = allHdrs.reduce((p, n) => { - if (seenHdrs[n.id]) - return p; - seenHdrs[n.id] = true; - return [...p, n] - }, []) - return res; + return unique(...all.map(ws => ws.listAsync)) } async function getAsync(h: Header): Promise { // chose the first matching one @@ -51,4 +58,80 @@ export function createJointWorkspace(primary: WorkspaceProvider, ...others: Work resetAsync, } return provider; +} + +export function createJointWorkspace(...all: CachedWorkspaceProvider[]): CachedWorkspaceProvider { + // TODO @darzu: we're assuming they are disjoint for now + + // TODO @darzu: debug logging + console.log(`createJointWorkspace`); + + const firstSync = async () => (await Promise.all(all.map(w => w.firstSync()))).reduce((p, n) => p || n, false) + const pendingSync = async () => (await Promise.all(all.map(w => w.pendingSync()))).reduce((p, n) => p || n, false) + const getLastModTime = () => Math.max(...all.map(w => w.getLastModTime())) + + async function synchronize(expectedLastModTime?: number): Promise { + return (await Promise.all(all.map(w => w.synchronize()))) + .reduce((p, n) => p || n, false) + } + function listSync(): Header[] { + // return all (assuming disjoint) + return all.map(w => w.listSync()) + .reduce((p, n) => [...p, ...n], []) + } + async function listAsync(): Promise { + await pendingSync() + // return all (assuming disjoint) + return (await Promise.all(all.map(w => w.listAsync()))) + .reduce((p, n) => [...p, ...n], []) + } + function getWorkspaceFor(h: Header): CachedWorkspaceProvider { + return all.reduce((p, n) => p || n.hasSync(h) ? n : p, null) + } + async function getAsync(h: Header): Promise { + await pendingSync() + // chose the first matching one + const ws = getWorkspaceFor(h) ?? all[0] + return ws.getAsync(h) + } + function tryGetSync(h: Header): File { + // chose the first matching one + const ws = getWorkspaceFor(h) ?? all[0] + return ws.tryGetSync(h) + } + function hasSync(h: Header): boolean { + return all.reduce((p, n) => p || n.hasSync(h), false) + } + async function setAsync(h: Header, prevVer: any, text?: ScriptText): Promise { + await pendingSync() + const ws = getWorkspaceFor(h) ?? all[0] + return ws.setAsync(h, prevVer, text) + } + async function deleteAsync(h: Header, prevVer: any): Promise { + await pendingSync() + const ws = getWorkspaceFor(h) + return ws?.deleteAsync(h, prevVer) + } + async function resetAsync() { + await pendingSync() + await Promise.all(all.map(ws => ws.resetAsync())) + } + + const provider: CachedWorkspaceProvider = { + // cache + getLastModTime, + synchronize, + pendingSync, + firstSync, + listSync, + hasSync, + tryGetSync, + // workspace + getAsync, + setAsync, + deleteAsync, + listAsync, + resetAsync, + } + return provider; } \ No newline at end of file diff --git a/webapp/src/workspaces/workspace.ts b/webapp/src/workspaces/workspace.ts index 3bcda441aaf..34ecbe7a56e 100644 --- a/webapp/src/workspaces/workspace.ts +++ b/webapp/src/workspaces/workspace.ts @@ -18,11 +18,12 @@ import * as cloudWorkspace from "./cloudworkspace" import U = pxt.Util; import Cloud = pxt.Cloud; -import { createJointWorkspace } from "./jointworkspace"; +import { createJointWorkspace, createJointWorkspace2 } from "./jointworkspace"; import { createBrowserDbWorkspace } from "./browserdbworkspace"; import { createSynchronizedWorkspace, Synchronizable } from "./synchronizedworkspace"; -import { ConflictStrategy, DisjointSetsStrategy, wrapInMemCache } from "./workspacebehavior"; +import { ConflictStrategy, DisjointSetsStrategy, migrateOverlap, wrapInMemCache } from "./workspacebehavior"; import { createMemWorkspace, SyncWorkspaceProvider } from "./memworkspace"; +import { CachedWorkspaceProvider, createCachedWorkspace, createCloudSyncWorkspace } from "./cloudsyncworkspace"; // Avoid importing entire crypto-js /* tslint:disable:no-submodule-imports */ @@ -40,7 +41,7 @@ let allScripts: File[] = []; let headerQ = new U.PromiseQueue(); let impl: WorkspaceProvider; -let implCache: SyncWorkspaceProvider & Synchronizable; +let implCache: CachedWorkspaceProvider; let implType: WorkspaceKind; function lookup(id: string): File { @@ -102,37 +103,48 @@ export function setupWorkspace(kind: WorkspaceKind): void { // TODO @darzu: console.log(`choosing workspace: ${kind}`); implType = kind ?? "browser"; - const localUserChoice = chooseWorkspace(implType); + const localChoice = chooseWorkspace(implType); // TODO @darzu: if (auth.loggedInSync()) { - const localCloud = createBrowserDbWorkspace("cloud-local"); - const cachedCloud = createSynchronizedWorkspace(cloudWorkspace.provider, localCloud, { - conflict: ConflictStrategy.LastWriteWins, - disjointSets: DisjointSetsStrategy.Synchronize - }); + const cloudApis = cloudWorkspace.provider + const localCloud = createBrowserDbWorkspace("cloud-local"); // TODO @darzu: use user choice for this too? + // TODO @darzu: + // const cachedCloud = createSynchronizedWorkspace(cloudWorkspace.provider, localCloud, { + // conflict: ConflictStrategy.LastWriteWins, + // disjointSets: DisjointSetsStrategy.Synchronize + // }); + const cloudCache = createCloudSyncWorkspace(cloudApis, localCloud); + + const localChoiceCached = createCachedWorkspace(localChoice) + + // TODO @darzu: do one-time overlap migration + // migrateOverlap(localChoiceCached, cloudCache) // TODO @darzu: improve this const msPerMin = 1000 * 60 const doSync = async () => { console.log("synchronizing with the cloud..."); - const changes = await cachedCloud.syncAsync() + const changes = await cloudCache.synchronize() console.log(`...changes synced! ${changes}`) } setInterval(doSync, 5 * msPerMin) + const joint = createJointWorkspace(cloudCache, localChoiceCached) + impl = joint + implCache = joint + // TODO @darzu: when synchronization causes changes // data.invalidate("header:*"); // data.invalidate("text:*"); // TODO @darzu: we are assuming these workspaces don't overlapp... - impl = createJointWorkspace(cachedCloud, localUserChoice) + // impl = createJointWorkspace2(cachedCloud, localChoice) + } + else { + // TODO @darzu: review + impl = localChoice + implCache = createCachedWorkspace(impl) } - else - impl = localUserChoice - - // TODO @darzu: remove allHeaders etc.. - implCache = wrapInMemCache(impl) - implCache.syncAsync(); // TODO @darzu: // if (changes.length) { diff --git a/webapp/src/workspaces/workspacebehavior.ts b/webapp/src/workspaces/workspacebehavior.ts index 3c48b8c7c02..54bd7338710 100644 --- a/webapp/src/workspaces/workspacebehavior.ts +++ b/webapp/src/workspaces/workspacebehavior.ts @@ -71,6 +71,7 @@ export interface SyncResult { right: Header[], } +// TODO @darzu: this has been moved into cloudsync workspace.. not sure it's still needed here export async function synchronize(left: WorkspaceProvider, right: WorkspaceProvider, strat: Strategy): Promise { // TODO @darzu: add "on changes identified" handler so we can show in-progress syncing @@ -139,4 +140,8 @@ export function wrapInMemCache(ws: WorkspaceProvider): SyncWorkspaceProvider & W conflict: ConflictStrategy.LastWriteWins, disjointSets: DisjointSetsStrategy.Synchronize }); +} + +export async function migrateOverlap(fromWs: WorkspaceProvider, toWs: WorkspaceProvider) { + // TODO @darzu: } \ No newline at end of file From f8201ce78f4bce92f2529fb25a00f2390ea3b148 Mon Sep 17 00:00:00 2001 From: darzu Date: Mon, 14 Dec 2020 11:36:05 -0800 Subject: [PATCH 30/52] fix build issue --- webapp/src/workspaces/cloudsyncworkspace.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/webapp/src/workspaces/cloudsyncworkspace.ts b/webapp/src/workspaces/cloudsyncworkspace.ts index e25419e93d2..f762be7e007 100644 --- a/webapp/src/workspaces/cloudsyncworkspace.ts +++ b/webapp/src/workspaces/cloudsyncworkspace.ts @@ -1,4 +1,3 @@ -import { del } from "request"; import { ConflictStrategy, DisjointSetsStrategy, Strategy } from "./workspacebehavior"; import U = pxt.Util; From 6939562243d56c75ef0ca15560b6f65f2aa82c4a Mon Sep 17 00:00:00 2001 From: darzu Date: Mon, 14 Dec 2020 11:45:07 -0800 Subject: [PATCH 31/52] fix lint issues --- pxtlib/util.ts | 2 +- webapp/src/workspaces/browserdbworkspace.ts | 4 +-- webapp/src/workspaces/cloudsync.ts | 4 +-- webapp/src/workspaces/cloudsyncworkspace.ts | 30 ++++++++++----------- webapp/src/workspaces/jointworkspace.ts | 10 +++---- webapp/src/workspaces/workspace.ts | 24 ++++++++--------- webapp/src/workspaces/workspacebehavior.ts | 10 +++---- 7 files changed, 42 insertions(+), 42 deletions(-) diff --git a/pxtlib/util.ts b/pxtlib/util.ts index 00afd8aa7f4..ea97c6b0599 100644 --- a/pxtlib/util.ts +++ b/pxtlib/util.ts @@ -1411,7 +1411,7 @@ namespace ts.pxtc.Util { } export function unreachable(...ns: never[]): never { - throw "Type error: this code should be unreachable"; + throw new Error("Type error: this code should be unreachable"); } } diff --git a/webapp/src/workspaces/browserdbworkspace.ts b/webapp/src/workspaces/browserdbworkspace.ts index 79425cf4732..86c6af1d0bd 100644 --- a/webapp/src/workspaces/browserdbworkspace.ts +++ b/webapp/src/workspaces/browserdbworkspace.ts @@ -21,7 +21,7 @@ export function createBrowserDbWorkspace(namespace: string): BrowserDbWorkspaceP // TODO @darzu: debug logging (async () => { const hdrs: pxt.workspace.Header[] = await headerDb.getAllAsync(); - const txts: TextDbEntry[] = await textDb.getAllAsync(); + const txts: TextDbEntry[] = await textDb.getAllAsync(); console.log(`createBrowserDbWorkspace: ${prefix}:`); console.dir(hdrs) console.dir(txts) @@ -51,7 +51,7 @@ export function createBrowserDbWorkspace(namespace: string): BrowserDbWorkspaceP console.log("setAsync with text :)") } - const textEnt: TextDbEntry = { + const textEnt: TextDbEntry = { id: h.id, files: text, _rev: prevVer diff --git a/webapp/src/workspaces/cloudsync.ts b/webapp/src/workspaces/cloudsync.ts index a2a643ab6ba..03fe7bc18f6 100644 --- a/webapp/src/workspaces/cloudsync.ts +++ b/webapp/src/workspaces/cloudsync.ts @@ -504,8 +504,8 @@ export function refreshToken() { export function syncAsync(): Promise { return Promise.all([ - githubSyncAsync(), - // TODO @darzu: + githubSyncAsync(), + // TODO @darzu: // cloud.syncAsync() cloudSyncAsync() ]) diff --git a/webapp/src/workspaces/cloudsyncworkspace.ts b/webapp/src/workspaces/cloudsyncworkspace.ts index f762be7e007..040a4449cb5 100644 --- a/webapp/src/workspaces/cloudsyncworkspace.ts +++ b/webapp/src/workspaces/cloudsyncworkspace.ts @@ -7,7 +7,7 @@ type F = pxt.workspace.File; type Version = pxt.workspace.Version; type WorkspaceProvider = pxt.workspace.WorkspaceProvider; -// TODO @darzu: +// TODO @darzu: // cache invalidation export interface CachedWorkspaceProvider extends WorkspaceProvider { @@ -33,7 +33,7 @@ export function createCachedWorkspace(ws: WorkspaceProvider): CachedWorkspacePro let cacheHdrs: Header[] = [] let cacheHdrsMap: {[id: string]: Header} = {}; let cacheProjs: {[id: string]: F} = {}; - + let cacheModTime: number = 0; function getLastModTime(): number { @@ -90,10 +90,10 @@ export function createCachedWorkspace(ws: WorkspaceProvider): CachedWorkspacePro } // save results - cacheModTime = newLastModTime + cacheModTime = newLastModTime cacheProjs = newProjs cacheHdrs = newHdrs - cacheHdrsMap = newHdrsMap + cacheHdrsMap = newHdrsMap return true; } @@ -116,7 +116,7 @@ export function createCachedWorkspace(ws: WorkspaceProvider): CachedWorkspacePro cacheProjs[h.id] = { header: h, text, - version: null // TODO @darzu: + version: null // TODO @darzu: } cacheModTime = Math.max(cacheModTime, h.modificationTime) const res = await ws.setAsync(h, prevVer, text) @@ -127,7 +127,7 @@ export function createCachedWorkspace(ws: WorkspaceProvider): CachedWorkspacePro delete cacheProjs[h.id]; // TODO @darzu: how to handle mod time with delete? // TODO @darzu: we should probably enforce soft delete everywhere... - cacheModTime = Math.max(cacheModTime, h.modificationTime) + cacheModTime = Math.max(cacheModTime, h.modificationTime) const res = await ws.deleteAsync(h, prevVer) return res; } @@ -187,7 +187,7 @@ export function createCloudSyncWorkspace(cloud: WorkspaceProvider, cloudLocal: W } async function transfer(h: Header, fromWs: WorkspaceProvider, toWs: WorkspaceProvider): Promise
{ const fromPrj = await fromWs.getAsync(h) - + // TODO @darzu: caches? const prevVersion: Version = null // TODO @darzu: what do we do with this version thing... @@ -217,12 +217,12 @@ export function createCloudSyncWorkspace(cloud: WorkspaceProvider, cloudLocal: W // TODO @darzu: short circuit if there aren't changes ? const lHdrsList = await left.listAsync() - const rHdrsList = await right.listAsync() + const rHdrsList = await right.listAsync() const lHdrs = U.toDictionary(lHdrsList, h => h.id) const rHdrs = U.toDictionary(rHdrsList, h => h.id) const allHdrsList = [...lHdrsList, ...rHdrsList] - + // determine left-only, overlap, and right-only sets const overlap = allHdrsList.reduce( (p: {[key: string]: Header}, n) => lHdrs[n.id] && rHdrs[n.id] ? (p[n.id] = n) && p : p, {}) @@ -230,28 +230,28 @@ export function createCloudSyncWorkspace(cloud: WorkspaceProvider, cloudLocal: W (p: {[key: string]: Header}, n) => lHdrs[n.id] && !rHdrs[n.id] ? (p[n.id] = n) && p : p, {}) const rOnly = allHdrsList.reduce( (p: {[key: string]: Header}, n) => !lHdrs[n.id] && rHdrs[n.id] ? (p[n.id] = n) && p : p, {}) - + // resolve conflicts const conflictResults = U.values(overlap).map(h => resolveConflict(lHdrs[h.id], rHdrs[h.id], strat.conflict)) - + // update left const lChanges = conflictResults.reduce((p: Header[], n) => hasChanged(n, lHdrs[n.id]) ? [...p, n] : p, []) let lToPush = lChanges if (strat.disjointSets === DisjointSetsStrategy.Synchronize) lToPush = [...lToPush, ...U.values(rOnly)] const lPushPromises = lToPush.map(h => transfer(h, right, left)) - + // update right const rChanges = conflictResults.reduce((p: Header[], n) => hasChanged(n, rHdrs[n.id]) ? [...p, n] : p, []) let rToPush = rChanges if (strat.disjointSets === DisjointSetsStrategy.Synchronize) rToPush = [...rToPush, ...U.values(lOnly)] const rPushPromises = rToPush.map(h => transfer(h, left, right)) - + // wait // TODO @darzu: batching? throttling? incremental? const changed = await Promise.all([...lPushPromises, ...rPushPromises]) - + // TODO @darzu: what about mod time changes? return changed.length >= 0; } @@ -283,7 +283,7 @@ export function createCloudSyncWorkspace(cloud: WorkspaceProvider, cloudLocal: W // await Promise.all([cloudCache.resetAsync(), localCache.resetAsync()]) return Promise.resolve(); } - + const provider: CloudSyncWorkspace = { // cache getLastModTime, diff --git a/webapp/src/workspaces/jointworkspace.ts b/webapp/src/workspaces/jointworkspace.ts index ebbd589a5a4..47b87911771 100644 --- a/webapp/src/workspaces/jointworkspace.ts +++ b/webapp/src/workspaces/jointworkspace.ts @@ -44,7 +44,7 @@ export function createJointWorkspace2(primary: WorkspaceProvider, ...others: Wor } async function deleteAsync(h: Header, prevVer: any): Promise { const matchingWorkspace = await getWorkspaceForAsync(h) - return matchingWorkspace?.deleteAsync(h, prevVer) + return matchingWorkspace?.deleteAsync(h, prevVer) } async function resetAsync() { await Promise.all(all.map(ws => ws.resetAsync())) @@ -62,7 +62,7 @@ export function createJointWorkspace2(primary: WorkspaceProvider, ...others: Wor export function createJointWorkspace(...all: CachedWorkspaceProvider[]): CachedWorkspaceProvider { // TODO @darzu: we're assuming they are disjoint for now - + // TODO @darzu: debug logging console.log(`createJointWorkspace`); @@ -71,7 +71,7 @@ export function createJointWorkspace(...all: CachedWorkspaceProvider[]): CachedW const getLastModTime = () => Math.max(...all.map(w => w.getLastModTime())) async function synchronize(expectedLastModTime?: number): Promise { - return (await Promise.all(all.map(w => w.synchronize()))) + return (await Promise.all(all.map(w => w.synchronize()))) .reduce((p, n) => p || n, false) } function listSync(): Header[] { @@ -108,9 +108,9 @@ export function createJointWorkspace(...all: CachedWorkspaceProvider[]): CachedW return ws.setAsync(h, prevVer, text) } async function deleteAsync(h: Header, prevVer: any): Promise { - await pendingSync() + await pendingSync() const ws = getWorkspaceFor(h) - return ws?.deleteAsync(h, prevVer) + return ws?.deleteAsync(h, prevVer) } async function resetAsync() { await pendingSync() diff --git a/webapp/src/workspaces/workspace.ts b/webapp/src/workspaces/workspace.ts index 34ecbe7a56e..f1f06960088 100644 --- a/webapp/src/workspaces/workspace.ts +++ b/webapp/src/workspaces/workspace.ts @@ -45,7 +45,7 @@ let implCache: CachedWorkspaceProvider; let implType: WorkspaceKind; function lookup(id: string): File { - // TODO @darzu: + // TODO @darzu: // const hdr = implCache.listSync().find(x => x.header.id == id || x.header.path == id) // implCache.getSync(); return allScripts.find(x => x.header.id == id || x.header.path == id); @@ -87,7 +87,7 @@ function chooseWorkspace(kind: WorkspaceKind = "browser"): pxt.workspace.Workspa case "idb": return indexedDBWorkspace.provider; case "cloud": - // TODO @darzu: + // TODO @darzu: console.log("CHOOSING CLOUD WORKSPACE"); return cloudWorkspace.provider; case "browser": @@ -100,15 +100,15 @@ function chooseWorkspace(kind: WorkspaceKind = "browser"): pxt.workspace.Workspa export function setupWorkspace(kind: WorkspaceKind): void { U.assert(!impl, "workspace set twice"); pxt.log(`workspace: ${kind}`); - // TODO @darzu: + // TODO @darzu: console.log(`choosing workspace: ${kind}`); implType = kind ?? "browser"; const localChoice = chooseWorkspace(implType); - // TODO @darzu: + // TODO @darzu: if (auth.loggedInSync()) { const cloudApis = cloudWorkspace.provider const localCloud = createBrowserDbWorkspace("cloud-local"); // TODO @darzu: use user choice for this too? - // TODO @darzu: + // TODO @darzu: // const cachedCloud = createSynchronizedWorkspace(cloudWorkspace.provider, localCloud, { // conflict: ConflictStrategy.LastWriteWins, // disjointSets: DisjointSetsStrategy.Synchronize @@ -146,18 +146,18 @@ export function setupWorkspace(kind: WorkspaceKind): void { implCache = createCachedWorkspace(impl) } - // TODO @darzu: + // TODO @darzu: // if (changes.length) { // data.invalidate("header:*"); // data.invalidate("text:*"); // } -} +} // TODO @darzu: needed? export function switchToCloudWorkspace(): string { U.assert(implType !== "cloud", "workspace already cloud"); const prevType = implType; - // TODO @darzu: + // TODO @darzu: console.log("switchToCloudWorkspace") impl = cloudWorkspace.provider; implType = "cloud"; @@ -212,9 +212,9 @@ export function getHeaders(withDeleted = false) { // let r = allScripts.map(e => e.header) let r = implCache.listSync() .filter(h => - (withDeleted || !h.isDeleted) + (withDeleted || !h.isDeleted) && !h.isBackup - // TODO @darzu: + // TODO @darzu: // && (!h.cloudUserId || h.cloudUserId === cloudUserId) ) r.sort((a, b) => b.recentUse - a.recentUse) @@ -1480,7 +1480,7 @@ export function syncAsync(): Promise { ex.text = undefined ex.version = undefined // TODO @darzu: handle data API subscriptions on header changed - console.log(`INVALIDATIN header ${hd.id}`) // TODO @darzu: + console.log(`INVALIDATIN header ${hd.id}`) // TODO @darzu: data.invalidateHeader("header", hd); data.invalidateHeader("text", hd); data.invalidateHeader("pkg-git-status", hd); @@ -1577,7 +1577,7 @@ data.mountVirtualApi("headers", { p = data.stripProtocol(p) const headers = getHeaders() if (!p) return Promise.resolve(headers) - console.log(`data SEARCH headers:${p}`) // TODO @darzu: + console.log(`data SEARCH headers:${p}`) // TODO @darzu: return compiler.projectSearchAsync({ term: p, headers }) .then((searchResults: pxtc.service.ProjectSearchInfo[]) => searchResults) .then(searchResults => { diff --git a/webapp/src/workspaces/workspacebehavior.ts b/webapp/src/workspaces/workspacebehavior.ts index 54bd7338710..20d897d511e 100644 --- a/webapp/src/workspaces/workspacebehavior.ts +++ b/webapp/src/workspaces/workspacebehavior.ts @@ -79,7 +79,7 @@ export async function synchronize(left: WorkspaceProvider, right: WorkspaceProvi // idea: never delete, only say "isDeleted" is true; can optimize away later /* sync scenarios: - cloud & cloud cache (last write wins; + cloud & cloud cache (last write wins; any workspace & memory workspace (memory leads) synchronization strategies: @@ -89,7 +89,7 @@ export async function synchronize(left: WorkspaceProvider, right: WorkspaceProvi primary & secondary primary always truth ? */ - + const lHdrsList = await left.listAsync() const rHdrsList = await right.listAsync() const lHdrs = U.toDictionary(lHdrsList, h => h.id) @@ -106,14 +106,14 @@ export async function synchronize(left: WorkspaceProvider, right: WorkspaceProvi // resolve conflicts const conflictResults = U.values(overlap).map(h => resolveConflict(lHdrs[h.id], rHdrs[h.id], strat.conflict)) - + // update left const lChanges = conflictResults.reduce((p: Header[], n) => hasChanged(n, lHdrs[n.id]) ? [...p, n] : p, []) let lToPush = lChanges if (strat.disjointSets === DisjointSetsStrategy.Synchronize) lToPush = [...lToPush, ...U.values(rOnly)] const lPushPromises = lToPush.map(h => transfer(h, right, left)) - + // update right const rChanges = conflictResults.reduce((p: Header[], n) => hasChanged(n, rHdrs[n.id]) ? [...p, n] : p, []) let rToPush = rChanges @@ -143,5 +143,5 @@ export function wrapInMemCache(ws: WorkspaceProvider): SyncWorkspaceProvider & W } export async function migrateOverlap(fromWs: WorkspaceProvider, toWs: WorkspaceProvider) { - // TODO @darzu: + // TODO @darzu: } \ No newline at end of file From e6c3f9b6f1d3ce53b7d8342d0cae06e2196d744c Mon Sep 17 00:00:00 2001 From: darzu Date: Tue, 15 Dec 2020 16:25:51 -0800 Subject: [PATCH 32/52] lots of debugging --- webapp/src/cloud.ts | 15 +++-- webapp/src/db.ts | 5 +- webapp/src/workspaces/browserdbworkspace.ts | 28 +++++---- webapp/src/workspaces/cloudsyncworkspace.ts | 13 ++++ webapp/src/workspaces/workspace.ts | 68 ++++++++++++++------- 5 files changed, 85 insertions(+), 44 deletions(-) diff --git a/webapp/src/cloud.ts b/webapp/src/cloud.ts index 2697c79520b..c0434ea2dff 100644 --- a/webapp/src/cloud.ts +++ b/webapp/src/cloud.ts @@ -27,7 +27,7 @@ export async function listAsync(): Promise { return new Promise(async (resolve, reject) => { // TODO @darzu: this is causing errors? const result = await auth.apiAsync("/api/user/project"); - console.log("cloud.ts:listAsync"); // TODO @darzu: + console.log("cloud.ts:listAsync"); // TODO @darzu: if (result.success) { const userId = auth.user()?.id; const headers = result.resp.map(proj => { @@ -37,7 +37,6 @@ export async function listAsync(): Promise { header.cloudCurrent = true; return header; }); - console.dir(headers) // TODO @darzu: resolve(headers); } else { reject(new Error(result.errmsg)); @@ -46,7 +45,7 @@ export async function listAsync(): Promise { } export function getAsync(h: Header): Promise { - console.log(`cloud.ts:getAsync ${h.id}`); // TODO @darzu: + console.log(`cloud.ts:getAsync ${h.id}`); // TODO @darzu: return new Promise(async (resolve, reject) => { const result = await auth.apiAsync(`/api/user/project/${h.id}`); if (result.success) { @@ -72,7 +71,7 @@ export function getAsync(h: Header): Promise { // TODO @darzu: is it okay to export this? export function setAsync(h: Header, prevVersion: Version, text?: ScriptText): Promise { - console.log(`cloud.ts:setAsync ${h.id}`); // TODO @darzu: + console.log(`cloud.ts:setAsync ${h.id}`); // TODO @darzu: return new Promise(async (resolve, reject) => { const userId = auth.user()?.id; h.cloudUserId = userId; @@ -84,7 +83,7 @@ export function setAsync(h: Header, prevVersion: Version, text?: ScriptText): Pr text: text ? JSON.stringify(text) : undefined, version: prevVersion } - // TODO @darzu: + // TODO @darzu: console.log("setAsync") const result = await auth.apiAsync('/api/user/project', project); if (result.success) { @@ -99,19 +98,19 @@ export function setAsync(h: Header, prevVersion: Version, text?: ScriptText): Pr } export function deleteAsync(h: Header, prevVersion: Version, text?: ScriptText): Promise { - console.log(`cloud.ts:deleteAsync ${h.id}`); // TODO @darzu: + console.log(`cloud.ts:deleteAsync ${h.id}`); // TODO @darzu: return Promise.resolve(); } export function resetAsync(): Promise { - console.log(`cloud.ts:resetAsync`); // TODO @darzu: + console.log(`cloud.ts:resetAsync`); // TODO @darzu: return Promise.resolve(); } export async function syncAsync(): Promise { if (!auth.hasIdentity()) { return; } if (!await auth.loggedIn()) { return; } - console.log(`cloud.ts:syncAsync`); // TODO @darzu: + console.log(`cloud.ts:syncAsync`); // TODO @darzu: try { const userId = auth.user()?.id; // Filter to cloud-synced headers owned by the current user. diff --git a/webapp/src/db.ts b/webapp/src/db.ts index 77b0a90bb7c..edb3060bede 100644 --- a/webapp/src/db.ts +++ b/webapp/src/db.ts @@ -14,7 +14,10 @@ const PouchDB = require("pouchdb") let _db: Promise = undefined; export function getDbAsync(): Promise { - if (_db) return _db; + if (_db) { + (window as any).db = _db + return _db; + } return _db = Promise.resolve() .then(() => { diff --git a/webapp/src/workspaces/browserdbworkspace.ts b/webapp/src/workspaces/browserdbworkspace.ts index 86c6af1d0bd..a9fdaa23956 100644 --- a/webapp/src/workspaces/browserdbworkspace.ts +++ b/webapp/src/workspaces/browserdbworkspace.ts @@ -18,20 +18,21 @@ export function createBrowserDbWorkspace(namespace: string): BrowserDbWorkspaceP const headerDb = new db.Table(`${prefix}header`); const textDb = new db.Table(`${prefix}text`); - // TODO @darzu: debug logging - (async () => { + // TODO @darzu: + const printDbg = async () => { const hdrs: pxt.workspace.Header[] = await headerDb.getAllAsync(); - const txts: TextDbEntry[] = await textDb.getAllAsync(); - console.log(`createBrowserDbWorkspace: ${prefix}:`); - console.dir(hdrs) - console.dir(txts) - })(); + // const txts: TextDbEntry[] = await textDb.getAllAsync(); + console.log(`${prefix}-headers:`); + console.dir(hdrs.map(h => ({id: h.id, t: h.modificationTime}))) + } + // TODO @darzu: dbg + printDbg(); async function listAsync(): Promise { const hdrs: pxt.workspace.Header[] = await headerDb.getAllAsync() - // TODO @darzu: debug logging - console.log(`browser db headers ${prefix}:`) - console.dir(hdrs.map(h => h.id)) + // // TODO @darzu: debug logging + // console.log(`browser db headers ${prefix}:`) + // console.dir(hdrs.map(h => h.id)) return hdrs } async function getAsync(h: Header): Promise { @@ -45,10 +46,10 @@ export function createBrowserDbWorkspace(namespace: string): BrowserDbWorkspaceP async function setAsync(h: Header, prevVer: any, text?: ScriptText): Promise { // TODO @darzu: debug logging if (!text) { - console.log("setAsync without text :(") + console.log("!!! setAsync without text :(") // console.dir(h) } else { - console.log("setAsync with text :)") + console.log(`setAsync ${namespace || "def"}:(${h.id}, ${h.modificationTime}) :)`) } const textEnt: TextDbEntry = { @@ -58,6 +59,9 @@ export function createBrowserDbWorkspace(namespace: string): BrowserDbWorkspaceP } const retrev = await textDb.setAsync(textEnt) const rev = await headerDb.setAsync(h) + + await printDbg(); // TODO @darzu: dbg + h._rev = rev return retrev } diff --git a/webapp/src/workspaces/cloudsyncworkspace.ts b/webapp/src/workspaces/cloudsyncworkspace.ts index 040a4449cb5..8f9a2e48790 100644 --- a/webapp/src/workspaces/cloudsyncworkspace.ts +++ b/webapp/src/workspaces/cloudsyncworkspace.ts @@ -58,6 +58,9 @@ export function createCachedWorkspace(ws: WorkspaceProvider): CachedWorkspacePro } async function synchronizeInternal(expectedLastModTime?:number): Promise { + + console.log("cachedworkspace: synchronizeInternal (1)") + // remember our old cache, we might keep items from it later const oldHdrs = cacheHdrs const oldHdrsMap = cacheHdrsMap @@ -77,6 +80,7 @@ export function createCachedWorkspace(ws: WorkspaceProvider): CachedWorkspacePro cacheHdrs = oldHdrs cacheHdrsMap = oldHdrsMap cacheProjs = oldProjs + console.log("cachedworkspace: synchronizeInternal (2)") return false } @@ -198,6 +202,8 @@ export function createCloudSyncWorkspace(cloud: WorkspaceProvider, cloudLocal: W async function synchronizeInternal(): Promise { await pendingCacheSync() + console.log("cloudsyncworkspace: synchronizeInternal") + if (cloudCache.getLastModTime() === localCache.getLastModTime()) { // we're synced up ? return false @@ -284,6 +290,13 @@ export function createCloudSyncWorkspace(cloud: WorkspaceProvider, cloudLocal: W return Promise.resolve(); } + + // TODO @darzu: debug logging + firstSync.then(c => { + console.log("first update:") + console.dir(localCache.listSync().map(h => ({id: h.id, t: h.modificationTime}))) + }) + const provider: CloudSyncWorkspace = { // cache getLastModTime, diff --git a/webapp/src/workspaces/workspace.ts b/webapp/src/workspaces/workspace.ts index f1f06960088..e92354523af 100644 --- a/webapp/src/workspaces/workspace.ts +++ b/webapp/src/workspaces/workspace.ts @@ -40,8 +40,9 @@ let allScripts: File[] = []; let headerQ = new U.PromiseQueue(); -let impl: WorkspaceProvider; -let implCache: CachedWorkspaceProvider; +let impl: CachedWorkspaceProvider; +// TODO @darzu: +// let implCache: CachedWorkspaceProvider; let implType: WorkspaceKind; function lookup(id: string): File { @@ -107,7 +108,8 @@ export function setupWorkspace(kind: WorkspaceKind): void { // TODO @darzu: if (auth.loggedInSync()) { const cloudApis = cloudWorkspace.provider - const localCloud = createBrowserDbWorkspace("cloud-local"); // TODO @darzu: use user choice for this too? + // const localCloud = createBrowserDbWorkspace("cloud-local"); // TODO @darzu: use user choice for this too? + const localCloud = createBrowserDbWorkspace(""); // TODO @darzu: undo dbg // TODO @darzu: // const cachedCloud = createSynchronizedWorkspace(cloudWorkspace.provider, localCloud, { // conflict: ConflictStrategy.LastWriteWins, @@ -120,18 +122,34 @@ export function setupWorkspace(kind: WorkspaceKind): void { // TODO @darzu: do one-time overlap migration // migrateOverlap(localChoiceCached, cloudCache) + const joint = createJointWorkspace(cloudCache, localChoiceCached) + impl = joint + impl = cloudCache // TODO @darzu: undo dbg + // implCache = joint + // TODO @darzu: improve this const msPerMin = 1000 * 60 + const afterSync = (changed: boolean) => { + console.log(`...changes synced! ${changed}`) + if (changed) { + data.invalidate("header:*"); + data.invalidate("text:*"); + } + } const doSync = async () => { console.log("synchronizing with the cloud..."); - const changes = await cloudCache.synchronize() - console.log(`...changes synced! ${changes}`) + console.log("before:") + console.dir(joint.listSync().map(j => ({id: j.id, t: j.modificationTime}))) + const changed = await joint.synchronize() + if (changed) { + console.log("after:") + console.dir(joint.listSync().map(j => ({id: j.id, t: j.modificationTime}))) + } + afterSync(changed) } setInterval(doSync, 5 * msPerMin) - - const joint = createJointWorkspace(cloudCache, localChoiceCached) - impl = joint - implCache = joint + // TODO @darzu: + joint.firstSync().then(afterSync) // TODO @darzu: when synchronization causes changes // data.invalidate("header:*"); @@ -142,8 +160,8 @@ export function setupWorkspace(kind: WorkspaceKind): void { } else { // TODO @darzu: review - impl = localChoice - implCache = createCachedWorkspace(impl) + const localWs = localChoice + impl = createCachedWorkspace(localWs) } // TODO @darzu: @@ -154,15 +172,15 @@ export function setupWorkspace(kind: WorkspaceKind): void { } // TODO @darzu: needed? -export function switchToCloudWorkspace(): string { - U.assert(implType !== "cloud", "workspace already cloud"); - const prevType = implType; - // TODO @darzu: - console.log("switchToCloudWorkspace") - impl = cloudWorkspace.provider; - implType = "cloud"; - return prevType; -} +// export function switchToCloudWorkspace(): string { +// U.assert(implType !== "cloud", "workspace already cloud"); +// const prevType = implType; +// // TODO @darzu: +// console.log("switchToCloudWorkspace") +// impl = cloudWorkspace.provider; +// implType = "cloud"; +// return prevType; +// } // TODO @darzu: needed? export function switchToWorkspace(id: WorkspaceKind) { @@ -197,7 +215,7 @@ async function switchToMemoryWorkspace(reason: string): Promise { }); } - impl = memoryworkspace.provider; + impl = createCachedWorkspace(memoryworkspace.provider); // TODO @darzu: use our new mem workspace implType = "mem"; } @@ -210,7 +228,7 @@ export function getHeaders(withDeleted = false) { const cloudUserId = auth.user()?.id; // TODO @darzu: use allScripts still? // let r = allScripts.map(e => e.header) - let r = implCache.listSync() + let r = impl.listSync() .filter(h => (withDeleted || !h.isDeleted) && !h.isBackup @@ -332,7 +350,8 @@ function checkHeaderSession(h: Header): void { export function initAsync() { if (!impl) { - impl = browserworkspace.provider; + // TODO @darzu: hmmmm we should be use setupWorkspace + impl = createCachedWorkspace(browserworkspace.provider); implType = "browser"; } @@ -494,6 +513,7 @@ export function saveAsync(h: Header, text?: ScriptText, isCloud?: boolean): Prom } catch (e) { // Write failed; use in memory db. // TODO @darzu: POLICY + console.log("switchToMemoryWorkspace (1)") // TODO @darzu: await switchToMemoryWorkspace("write failed"); ver = await impl.setAsync(h, e.version, toWrite); } @@ -1465,6 +1485,8 @@ export function syncAsync(): Promise { // There might be a problem with the native databases. Switch to memory for this session so the user can at // least use the editor. // TODO @darzu: POLICY + console.log("switchToMemoryWorkspace (2)") // TODO @darzu: + console.dir(e) return switchToMemoryWorkspace("sync failed") .then(() => impl.listAsync()); }) From 94605ea460319bb9674ae766a057e86723d15011 Mon Sep 17 00:00:00 2001 From: darzu Date: Tue, 15 Dec 2020 16:47:44 -0800 Subject: [PATCH 33/52] debugging --- .../src/workspaces/oldbrowserdbworkspace.ts | 127 ++++++++++++++++++ webapp/src/workspaces/workspace.ts | 8 ++ 2 files changed, 135 insertions(+) create mode 100644 webapp/src/workspaces/oldbrowserdbworkspace.ts diff --git a/webapp/src/workspaces/oldbrowserdbworkspace.ts b/webapp/src/workspaces/oldbrowserdbworkspace.ts new file mode 100644 index 00000000000..d7d39645e1f --- /dev/null +++ b/webapp/src/workspaces/oldbrowserdbworkspace.ts @@ -0,0 +1,127 @@ +import * as db from "../db"; + +let headers: db.Table; +let texts: db.Table; + +type Header = pxt.workspace.Header; +type ScriptText = pxt.workspace.ScriptText; +type WorkspaceProvider = pxt.workspace.WorkspaceProvider; + +function migratePrefixesAsync(): Promise { + const currentVersion = pxt.semver.parse(pxt.appTarget.versions.target); + const currentMajor = currentVersion.major; + const currentDbPrefix = pxt.appTarget.appTheme.browserDbPrefixes && pxt.appTarget.appTheme.browserDbPrefixes[currentMajor]; + + if (!currentDbPrefix) { + // This version does not use a prefix for storing projects, so just use default tables + headers = new db.Table("header"); + texts = new db.Table("text"); + return Promise.resolve(); + } + + headers = new db.Table(`${currentDbPrefix}-header`); + texts = new db.Table(`${currentDbPrefix}-text`); + + return headers.getAllAsync() + .then((allDbHeaders) => { + if (allDbHeaders.length) { + // There are already scripts using the prefix, so a migration has already happened + return Promise.resolve(); + } + + // No headers using this prefix yet, attempt to migrate headers from previous major version (or default tables) + const previousMajor = currentMajor - 1; + const previousDbPrefix = previousMajor < 0 ? "" : pxt.appTarget.appTheme.browserDbPrefixes && pxt.appTarget.appTheme.browserDbPrefixes[previousMajor]; + let previousHeaders = new db.Table("header"); + let previousTexts = new db.Table("text"); + + if (previousDbPrefix) { + previousHeaders = new db.Table(`${previousDbPrefix}-header`); + previousTexts = new db.Table(`${previousDbPrefix}-text`); + } + + const copyProject = (h: pxt.workspace.Header): Promise => { + return previousTexts.getAsync(h.id) + .then((resp) => { + // Ignore metadata of the previous script so they get re-generated for the new copy + delete (h)._id; + delete (h)._rev; + return setAsync(h, undefined, resp.files); + }); + }; + + return previousHeaders.getAllAsync() + .then((previousHeaders: pxt.workspace.Header[]) => { + return Promise.map(previousHeaders, (h) => copyProject(h)); + }) + .then(() => { }); + }); +} + +function listAsync(): Promise { + return migratePrefixesAsync() + .then(() => headers.getAllAsync()); +} + +function getAsync(h: Header): Promise { + return texts.getAsync(h.id) + .then(resp => ({ + header: h, + text: resp.files, + version: resp._rev + })); +} + +function setAsync(h: Header, prevVer: any, text?: ScriptText) { + return setCoreAsync(headers, texts, h, prevVer, text); +} + +function setCoreAsync(headers: db.Table, texts: db.Table, h: Header, prevVer: any, text?: ScriptText) { + let retrev = "" + return (!text ? Promise.resolve() : + texts.setAsync({ + id: h.id, + files: text, + _rev: prevVer + }).then(rev => { + retrev = rev + })) + .then(() => headers.setAsync(h)) + .then(rev => { + h._rev = rev + return retrev + }); +} + +export function copyProjectToLegacyEditor(h: Header, majorVersion: number): Promise
{ + const prefix = pxt.appTarget.appTheme.browserDbPrefixes && pxt.appTarget.appTheme.browserDbPrefixes[majorVersion]; + + const oldHeaders = new db.Table(prefix ? `${prefix}-header` : `header`); + const oldTexts = new db.Table(prefix ? `${prefix}-text` : `text`); + + const header = pxt.Util.clone(h); + delete (header as any)._id; + delete header._rev; + header.id = pxt.Util.guidGen(); + + return getAsync(h) + .then(resp => setCoreAsync(oldHeaders, oldTexts, header, undefined, resp.text)) + .then(rev => header); +} + +function deleteAsync(h: Header, prevVer: any) { + return headers.deleteAsync(h) + .then(() => texts.deleteAsync({ id: h.id, _rev: h._rev })); +} + +function resetAsync() { + // workspace.resetAsync already clears all tables + return Promise.resolve(); +} +export const provider: WorkspaceProvider = { + getAsync, + setAsync, + deleteAsync, + listAsync, + resetAsync, +} \ No newline at end of file diff --git a/webapp/src/workspaces/workspace.ts b/webapp/src/workspaces/workspace.ts index e92354523af..85fd688ec81 100644 --- a/webapp/src/workspaces/workspace.ts +++ b/webapp/src/workspaces/workspace.ts @@ -15,6 +15,8 @@ import * as compiler from "../compiler" import * as auth from "../auth" import * as cloud from "../cloud" import * as cloudWorkspace from "./cloudworkspace" +import * as oldbrowserdbworkspace from "./oldbrowserdbworkspace" // TODO @darzu: dbg + import U = pxt.Util; import Cloud = pxt.Cloud; @@ -122,6 +124,12 @@ export function setupWorkspace(kind: WorkspaceKind): void { // TODO @darzu: do one-time overlap migration // migrateOverlap(localChoiceCached, cloudCache) + const old = oldbrowserdbworkspace.provider; + old.listAsync().then(hs => { + console.log("OLD:") + console.dir(hs.map(h => ({id: h.id, t: h.modificationTime}))) + }) + const joint = createJointWorkspace(cloudCache, localChoiceCached) impl = joint impl = cloudCache // TODO @darzu: undo dbg From 592c1ecd5a23ce5d31b41f29ddc7b54ee446f7a8 Mon Sep 17 00:00:00 2001 From: darzu Date: Wed, 16 Dec 2020 19:58:45 -0800 Subject: [PATCH 34/52] horrible debugging --- pxteditor/localStorage.ts | 1 + pxteditor/workspace.ts | 2 + webapp/src/app.tsx | 2 + webapp/src/cloud.ts | 8 +-- webapp/src/db.ts | 17 +++++- webapp/src/workspaces/browserdbworkspace.ts | 59 +++++++++++++++++---- webapp/src/workspaces/cloudsyncworkspace.ts | 42 ++++++++++----- webapp/src/workspaces/workspace.ts | 59 ++++++++++++--------- 8 files changed, 138 insertions(+), 52 deletions(-) diff --git a/pxteditor/localStorage.ts b/pxteditor/localStorage.ts index 48057dd3a37..5171cff1652 100644 --- a/pxteditor/localStorage.ts +++ b/pxteditor/localStorage.ts @@ -1,4 +1,5 @@ namespace pxt.storage { + // TODO @darzu: why is this different from the WorkspaceProvider api? interface IStorage { removeItem(key: string): void; getItem(key: string): string; diff --git a/pxteditor/workspace.ts b/pxteditor/workspace.ts index b3078cf356e..d0e5f92f14f 100644 --- a/pxteditor/workspace.ts +++ b/pxteditor/workspace.ts @@ -23,6 +23,8 @@ namespace pxt.workspace { export interface File { header: Header; text: ScriptText; + // This version field is reserved for the storage mechanism. E.g. PouchDB requires a _rev field containing + // the currently stored version. version: Version; } diff --git a/webapp/src/app.tsx b/webapp/src/app.tsx index 9a263c07bca..dff9412966a 100644 --- a/webapp/src/app.tsx +++ b/webapp/src/app.tsx @@ -3278,6 +3278,7 @@ export class ProjectView } showResetDialog() { + console.log("showResetDialog (1)") dialogs.showResetDialogAsync().done(r => { if (!r) return Promise.resolve(); return Promise.resolve() @@ -3285,6 +3286,7 @@ export class ProjectView return pxt.winrt.releaseAllDevicesAsync(); }) .then(() => { + console.log("showResetDialog (2)") return this.resetWorkspace(); }); }); diff --git a/webapp/src/cloud.ts b/webapp/src/cloud.ts index c0434ea2dff..dd1527726db 100644 --- a/webapp/src/cloud.ts +++ b/webapp/src/cloud.ts @@ -27,16 +27,18 @@ export async function listAsync(): Promise { return new Promise(async (resolve, reject) => { // TODO @darzu: this is causing errors? const result = await auth.apiAsync("/api/user/project"); - console.log("cloud.ts:listAsync"); // TODO @darzu: + console.log("cloud.ts:listAsync"); // TODO @darzu: dbg if (result.success) { const userId = auth.user()?.id; - const headers = result.resp.map(proj => { + const headers: Header[] = result.resp.map(proj => { const header = JSON.parse(proj.header); header.cloudUserId = userId; header.cloudVersion = proj.version; header.cloudCurrent = true; return header; }); + // TODO @darzu: dbg + console.dir(headers.map(h => ({h: h.id, t: h.modificationTime}))) resolve(headers); } else { reject(new Error(result.errmsg)); @@ -83,8 +85,6 @@ export function setAsync(h: Header, prevVersion: Version, text?: ScriptText): Pr text: text ? JSON.stringify(text) : undefined, version: prevVersion } - // TODO @darzu: - console.log("setAsync") const result = await auth.apiAsync('/api/user/project', project); if (result.success) { h.cloudCurrent = true; diff --git a/webapp/src/db.ts b/webapp/src/db.ts index edb3060bede..c9a18443998 100644 --- a/webapp/src/db.ts +++ b/webapp/src/db.ts @@ -33,9 +33,16 @@ export function getDbAsync(): Promise { } export function destroyAsync(): Promise { + console.log("destroying db! (1)") // TODO @darzu: dbg return !_db ? Promise.resolve() : _db.then((db: any) => { - db.destroy(); + console.log("destroying db! (2)") // TODO @darzu: dbg + const res: Promise = db.destroy() _db = undefined; + return res + }).then(r => { + console.log("destroy res") + console.dir(r) + return r }); } @@ -78,9 +85,15 @@ export class Table { .catch(e => { if (e.status == 409) { // conflict while writing key, ignore. - pxt.debug(`table: set conflict (409)`); + pxt.debug(`table: set conflict (409) for ${obj._id}#${obj._rev}`); return undefined; } + if (e.status == 400) { + // bad request; likely _rev format was wrong or something similiar + pxt.debug(`table: ${e.name}:${e.message} for ${obj._id}#${obj._rev}`); + // TODO @darzu: what's the right behavior here? Do we ever expect a 400 in normal operation? + // return undefined; + } pxt.reportException(e); pxt.log(`table: set failed, cleaning translation db`) // clean up translation and try again diff --git a/webapp/src/workspaces/browserdbworkspace.ts b/webapp/src/workspaces/browserdbworkspace.ts index a9fdaa23956..aca70812dfb 100644 --- a/webapp/src/workspaces/browserdbworkspace.ts +++ b/webapp/src/workspaces/browserdbworkspace.ts @@ -4,15 +4,17 @@ type Header = pxt.workspace.Header; type ScriptText = pxt.workspace.ScriptText; type TextDbEntry = { - id: string, files?: ScriptText, - _rev: any + // These are required by PouchDB/CouchDB + id: string, + _rev: any // This must be set to the return value of the last PouchDB/CouchDB } export interface BrowserDbWorkspaceProvider extends pxt.workspace.WorkspaceProvider { prefix: string; } +// TODO @darzu: very important for _rev and _id export function createBrowserDbWorkspace(namespace: string): BrowserDbWorkspaceProvider { const prefix = namespace ? namespace + "-" : "" const headerDb = new db.Table(`${prefix}header`); @@ -22,7 +24,7 @@ export function createBrowserDbWorkspace(namespace: string): BrowserDbWorkspaceP const printDbg = async () => { const hdrs: pxt.workspace.Header[] = await headerDb.getAllAsync(); // const txts: TextDbEntry[] = await textDb.getAllAsync(); - console.log(`${prefix}-headers:`); + console.log(`dbg ${prefix}-headers:`); console.dir(hdrs.map(h => ({id: h.id, t: h.modificationTime}))) } // TODO @darzu: dbg @@ -49,25 +51,62 @@ export function createBrowserDbWorkspace(namespace: string): BrowserDbWorkspaceP console.log("!!! setAsync without text :(") // console.dir(h) } else { - console.log(`setAsync ${namespace || "def"}:(${h.id}, ${h.modificationTime}) :)`) + console.log(`setAsync ${namespace || "def"}:(${h.id}, ${h.modificationTime}, ${prevVer}) :)`) } const textEnt: TextDbEntry = { - id: h.id, files: text, + id: h.id, _rev: prevVer } - const retrev = await textDb.setAsync(textEnt) - const rev = await headerDb.setAsync(h) + + // if we get a 400, we need to fetch the old then do a new + let textVer: string; + try { + textVer = await textDb.setAsync(textEnt) + } catch (e) {} + + if (!textVer) + console.log(`! failed to set text for id:${h.id},pv:${prevVer}`); // TODO @darzu: dbg logging + + let hdrVer: string; + try { + hdrVer = await headerDb.setAsync(h) + } catch (e) {} + + if (!hdrVer) { + console.log(`! failed to set hdr for id:${h.id},pv:${prevVer}`); // TODO @darzu: dbg logging + let oldHdr: Header + try { + oldHdr = await headerDb.getAsync(h.id) as Header + } catch (e) {} + if (oldHdr) { + h._rev = oldHdr._rev + } else { + delete h._rev + } + // TODO @darzu: need to rethink error handling here + try { + hdrVer = await headerDb.setAsync(h) + } catch (e) {} + if (!hdrVer) { + console.log(`!!! failed AGAIN to set hdr for id:${h.id},old:${JSON.stringify(oldHdr)}`); // TODO @darzu: dbg logging + } + } + + h._rev = hdrVer await printDbg(); // TODO @darzu: dbg - h._rev = rev - return retrev + return textVer } async function deleteAsync(h: Header, prevVer: any): Promise { await headerDb.deleteAsync(h) - await textDb.deleteAsync({ id: h.id, _rev: h._rev }) + const textEnt: TextDbEntry = { + id: h.id, + _rev: prevVer + } + await textDb.deleteAsync(textEnt); } async function resetAsync() { // workspace.resetAsync already clears all tables diff --git a/webapp/src/workspaces/cloudsyncworkspace.ts b/webapp/src/workspaces/cloudsyncworkspace.ts index 8f9a2e48790..933834db3c0 100644 --- a/webapp/src/workspaces/cloudsyncworkspace.ts +++ b/webapp/src/workspaces/cloudsyncworkspace.ts @@ -7,8 +7,11 @@ type F = pxt.workspace.File; type Version = pxt.workspace.Version; type WorkspaceProvider = pxt.workspace.WorkspaceProvider; -// TODO @darzu: -// cache invalidation +// TODO @darzu: BIG TODOs +// [ ] cache invalidation via header sessions +// [ ] enforce soft-delete +// pouchdb uses _delete for soft delete +// [ ] need to think more about error handling and retries export interface CachedWorkspaceProvider extends WorkspaceProvider { getLastModTime(): number, @@ -51,6 +54,7 @@ export function createCachedWorkspace(ws: WorkspaceProvider): CachedWorkspacePro } function eraseCache() { + console.log("cachedworkspace: eraseCache") // TODO @darzu: dbg cacheHdrs = [] cacheHdrsMap = {} cacheProjs = {} @@ -58,9 +62,6 @@ export function createCachedWorkspace(ws: WorkspaceProvider): CachedWorkspacePro } async function synchronizeInternal(expectedLastModTime?:number): Promise { - - console.log("cachedworkspace: synchronizeInternal (1)") - // remember our old cache, we might keep items from it later const oldHdrs = cacheHdrs const oldHdrsMap = cacheHdrsMap @@ -80,9 +81,9 @@ export function createCachedWorkspace(ws: WorkspaceProvider): CachedWorkspacePro cacheHdrs = oldHdrs cacheHdrsMap = oldHdrsMap cacheProjs = oldProjs - console.log("cachedworkspace: synchronizeInternal (2)") return false } + console.log("cachedworkspace: synchronizeInternal (1)") // TODO @darzu: dbg // compute header differences and clear old cache entries const newHdrsMap = U.toDictionary(newHdrs, h => h.id) @@ -120,7 +121,7 @@ export function createCachedWorkspace(ws: WorkspaceProvider): CachedWorkspacePro cacheProjs[h.id] = { header: h, text, - version: null // TODO @darzu: + version: prevVer } cacheModTime = Math.max(cacheModTime, h.modificationTime) const res = await ws.setAsync(h, prevVer, text) @@ -185,18 +186,35 @@ export function createCloudSyncWorkspace(cloud: WorkspaceProvider, cloudLocal: W } function resolveConflict(a: Header, b: Header, strat: ConflictStrategy): Header { + // TODO @darzu: involve the user + // TODO @darzu: consider lineage + // TODO @darzu: consider diff if (strat === ConflictStrategy.LastWriteWins) return a.modificationTime > b.modificationTime ? a : b; U.unreachable(strat); } async function transfer(h: Header, fromWs: WorkspaceProvider, toWs: WorkspaceProvider): Promise
{ + // TODO @darzu: worklist this? + // TODO @darzu: respect e-tags and versions + // TODO @darzu: track pending saves const fromPrj = await fromWs.getAsync(h) - // TODO @darzu: caches? - - const prevVersion: Version = null // TODO @darzu: what do we do with this version thing... - // TODO @darzu: track pending saves - const toRes: Version = await toWs.setAsync(h, prevVersion, fromPrj.text) + console.log(`transfer ${h.id}#${h._rev}`) // TODO @darzu: dbg + let newVer: Version + try { + newVer = await toWs.setAsync(h, fromPrj.version, fromPrj.text) + } catch (e) {} + + if (!newVer) { + // force an update ignore e-tags / versions + const old = await toWs.getAsync(h) + console.log(`retrying ${h.id} (newV: ${fromPrj.version}, oldV: ${old.version}, _conflict: ${(old as any)._conflict})`) // TODO @darzu: dbg + newVer = await toWs.setAsync(h, old.version, fromPrj.text) + if (!newVer) { + // TODO @darzu: dbg + console.log(`! transfer failed for ${h.id}`) + } + } return h; } async function synchronizeInternal(): Promise { diff --git a/webapp/src/workspaces/workspace.ts b/webapp/src/workspaces/workspace.ts index 85fd688ec81..01f86a96166 100644 --- a/webapp/src/workspaces/workspace.ts +++ b/webapp/src/workspaces/workspace.ts @@ -109,9 +109,10 @@ export function setupWorkspace(kind: WorkspaceKind): void { const localChoice = chooseWorkspace(implType); // TODO @darzu: if (auth.loggedInSync()) { + console.log("logged in") // TODO @darzu: const cloudApis = cloudWorkspace.provider - // const localCloud = createBrowserDbWorkspace("cloud-local"); // TODO @darzu: use user choice for this too? - const localCloud = createBrowserDbWorkspace(""); // TODO @darzu: undo dbg + const localCloud = createBrowserDbWorkspace("cloud-local"); // TODO @darzu: use user choice for this too? + // const localCloud = createBrowserDbWorkspace(""); // TODO @darzu: undo dbg // TODO @darzu: // const cachedCloud = createSynchronizedWorkspace(cloudWorkspace.provider, localCloud, { // conflict: ConflictStrategy.LastWriteWins, @@ -124,15 +125,16 @@ export function setupWorkspace(kind: WorkspaceKind): void { // TODO @darzu: do one-time overlap migration // migrateOverlap(localChoiceCached, cloudCache) - const old = oldbrowserdbworkspace.provider; - old.listAsync().then(hs => { - console.log("OLD:") - console.dir(hs.map(h => ({id: h.id, t: h.modificationTime}))) - }) + // TODO @darzu: dbg: + // const old = oldbrowserdbworkspace.provider; + // old.listAsync().then(hs => { + // console.log("OLD browser db:") + // console.dir(hs.map(h => ({id: h.id, t: h.modificationTime}))) + // }) const joint = createJointWorkspace(cloudCache, localChoiceCached) impl = joint - impl = cloudCache // TODO @darzu: undo dbg + // impl = cloudCache // TODO @darzu: undo dbg // implCache = joint // TODO @darzu: improve this @@ -167,6 +169,7 @@ export function setupWorkspace(kind: WorkspaceKind): void { // impl = createJointWorkspace2(cachedCloud, localChoice) } else { + console.log("logged in") // TODO @darzu: // TODO @darzu: review const localWs = localChoice impl = createCachedWorkspace(localWs) @@ -649,6 +652,7 @@ export function fixupFileNames(txt: ScriptText) { } +// TODO @darzu: do we need this raw table? might not have a browser db even const scriptDlQ = new U.PromiseQueue(); const scripts = new db.Table("script"); // cache for published scripts export async function getPublishedScriptAsync(id: string) { @@ -1536,24 +1540,31 @@ export function syncAsync(): Promise { }); } -export function resetAsync() { +export async function resetAsync() { // TODO @darzu: this should just pass through to workspace impl + console.log("resetAsync (1)") // TODO @darzu: allScripts = [] - return impl.resetAsync() - .then(cloudsync.resetAsync) - .then(db.destroyAsync) - .then(pxt.BrowserUtils.clearTranslationDbAsync) - .then(pxt.BrowserUtils.clearTutorialInfoDbAsync) - .then(compiler.clearApiInfoDbAsync) - .then(() => { - pxt.storage.clearLocal(); - data.clearCache(); - // keep local token (localhost and electron) on reset - if (Cloud.localToken) - pxt.storage.setLocal("local_token", Cloud.localToken); - }) - .then(() => syncAsync()) // sync again to notify other tabs - .then(() => { }); + await impl.resetAsync(); + console.log("resetAsync (2)") // TODO @darzu: + await cloudsync.resetAsync(); + console.log("resetAsync (3)") // TODO @darzu: + await db.destroyAsync(); + console.log("resetAsync (4)") // TODO @darzu: + await pxt.BrowserUtils.clearTranslationDbAsync(); + console.log("resetAsync (5)") // TODO @darzu: + await pxt.BrowserUtils.clearTutorialInfoDbAsync(); + console.log("resetAsync (6)") // TODO @darzu: + await compiler.clearApiInfoDbAsync(); + console.log("resetAsync (7)") // TODO @darzu: + pxt.storage.clearLocal(); + console.log("resetAsync (8)") // TODO @darzu: + data.clearCache(); + console.log("resetAsync (9)") // TODO @darzu: + // keep local token (localhost and electron) on reset + if (Cloud.localToken) + pxt.storage.setLocal("local_token", Cloud.localToken); + await syncAsync() // sync again to notify other tab; + console.log("resetAsync (10)") // TODO @darzu: } export function loadedAsync() { From 7971ff3d71476f4034fa38d978720acf7b8124f4 Mon Sep 17 00:00:00 2001 From: darzu Date: Thu, 17 Dec 2020 11:22:12 -0800 Subject: [PATCH 35/52] fix project transfer & debugging --- pxteditor/workspace.ts | 2 +- webapp/src/cloud.ts | 2 +- webapp/src/db.ts | 1 + webapp/src/workspaces/cloudsyncworkspace.ts | 46 ++++++++++----------- webapp/src/workspaces/workspace.ts | 2 +- 5 files changed, 27 insertions(+), 26 deletions(-) diff --git a/pxteditor/workspace.ts b/pxteditor/workspace.ts index d0e5f92f14f..3de8ae09ad4 100644 --- a/pxteditor/workspace.ts +++ b/pxteditor/workspace.ts @@ -16,7 +16,7 @@ namespace pxt.workspace { url: string; } - // TODO @darzu: why can version be "any" ? that's really annoying to reason about + // TODO @darzu: why is version "any" ? that's really annoying to reason about. used as: string | ScriptText // TODO @darzu: _rev is a string; modificationTime is an int export type Version = any; diff --git a/webapp/src/cloud.ts b/webapp/src/cloud.ts index dd1527726db..d466f037842 100644 --- a/webapp/src/cloud.ts +++ b/webapp/src/cloud.ts @@ -126,7 +126,7 @@ export async function syncAsync(): Promise { if (local.cloudCurrent) { // No local changes, download latest. const file = await getAsync(local); - workspace.saveAsync(file.header, file.text, file.version); + workspace.saveAsync(file.header, file.text, true); } else { // Conflict. // TODO: Figure out how to register these. diff --git a/webapp/src/db.ts b/webapp/src/db.ts index c9a18443998..57a1a5f1dd2 100644 --- a/webapp/src/db.ts +++ b/webapp/src/db.ts @@ -37,6 +37,7 @@ export function destroyAsync(): Promise { return !_db ? Promise.resolve() : _db.then((db: any) => { console.log("destroying db! (2)") // TODO @darzu: dbg const res: Promise = db.destroy() + res.then(_ => console.log("db destroyed")) // TODO @darzu: _db = undefined; return res }).then(r => { diff --git a/webapp/src/workspaces/cloudsyncworkspace.ts b/webapp/src/workspaces/cloudsyncworkspace.ts index 933834db3c0..6244120fc1d 100644 --- a/webapp/src/workspaces/cloudsyncworkspace.ts +++ b/webapp/src/workspaces/cloudsyncworkspace.ts @@ -27,7 +27,7 @@ function computeLastModTime(hdrs: Header[]): number { return hdrs.reduce((p, n) => Math.max(n.modificationTime, p), 0) } function hasChanged(a: Header, b: Header): boolean { - // TODO @darzu: use version uuid instead? + // TODO @darzu: use e-tag, _rev, or version uuid instead? return a.modificationTime !== b.modificationTime } @@ -193,29 +193,29 @@ export function createCloudSyncWorkspace(cloud: WorkspaceProvider, cloudLocal: W return a.modificationTime > b.modificationTime ? a : b; U.unreachable(strat); } - async function transfer(h: Header, fromWs: WorkspaceProvider, toWs: WorkspaceProvider): Promise
{ + async function transfer(fromH: Header, toH: Header, fromWs: WorkspaceProvider, toWs: WorkspaceProvider): Promise
{ // TODO @darzu: worklist this? - // TODO @darzu: respect e-tags and versions // TODO @darzu: track pending saves - const fromPrj = await fromWs.getAsync(h) - - console.log(`transfer ${h.id}#${h._rev}`) // TODO @darzu: dbg - let newVer: Version - try { - newVer = await toWs.setAsync(h, fromPrj.version, fromPrj.text) - } catch (e) {} - - if (!newVer) { - // force an update ignore e-tags / versions - const old = await toWs.getAsync(h) - console.log(`retrying ${h.id} (newV: ${fromPrj.version}, oldV: ${old.version}, _conflict: ${(old as any)._conflict})`) // TODO @darzu: dbg - newVer = await toWs.setAsync(h, old.version, fromPrj.text) - if (!newVer) { - // TODO @darzu: dbg - console.log(`! transfer failed for ${h.id}`) - } + + const newPrj = await fromWs.getAsync(fromH) + + // we need the old project if any exists so we know what prevVersion to pass + // TODO @darzu: keep project text version in the header + let prevVer = undefined + if (toH) { + const oldPrj = await toWs.getAsync(toH) + if (oldPrj) + prevVer = oldPrj.version } - return h; + + // create a new header + // TODO @darzu: how do we do this in an abstraction preserving way? + const newH = {...fromH, _rev: toH?._rev ?? undefined} + delete (newH as any)["_id"] + + const newVer = await toWs.setAsync(newH, prevVer, newPrj.text) + + return newH; } async function synchronizeInternal(): Promise { await pendingCacheSync() @@ -263,14 +263,14 @@ export function createCloudSyncWorkspace(cloud: WorkspaceProvider, cloudLocal: W let lToPush = lChanges if (strat.disjointSets === DisjointSetsStrategy.Synchronize) lToPush = [...lToPush, ...U.values(rOnly)] - const lPushPromises = lToPush.map(h => transfer(h, right, left)) + const lPushPromises = lToPush.map(h => transfer(rHdrs[h.id], lHdrs[h.id], right, left)) // update right const rChanges = conflictResults.reduce((p: Header[], n) => hasChanged(n, rHdrs[n.id]) ? [...p, n] : p, []) let rToPush = rChanges if (strat.disjointSets === DisjointSetsStrategy.Synchronize) rToPush = [...rToPush, ...U.values(lOnly)] - const rPushPromises = rToPush.map(h => transfer(h, left, right)) + const rPushPromises = rToPush.map(h => transfer(lHdrs[h.id], rHdrs[h.id], left, right)) // wait // TODO @darzu: batching? throttling? incremental? diff --git a/webapp/src/workspaces/workspace.ts b/webapp/src/workspaces/workspace.ts index 01f86a96166..dc66a998e11 100644 --- a/webapp/src/workspaces/workspace.ts +++ b/webapp/src/workspaces/workspace.ts @@ -169,7 +169,7 @@ export function setupWorkspace(kind: WorkspaceKind): void { // impl = createJointWorkspace2(cachedCloud, localChoice) } else { - console.log("logged in") // TODO @darzu: + console.log("logged out") // TODO @darzu: // TODO @darzu: review const localWs = localChoice impl = createCachedWorkspace(localWs) From 88717cffbb8444ba78a9cf716362eca39f93635e Mon Sep 17 00:00:00 2001 From: darzu Date: Fri, 18 Dec 2020 09:33:03 -0800 Subject: [PATCH 36/52] fixining sync --- webapp/src/db.ts | 2 +- webapp/src/workspaces/cloudsyncworkspace.ts | 84 ++++++++++++++++----- webapp/src/workspaces/jointworkspace.ts | 2 +- webapp/src/workspaces/workspace.ts | 3 +- 4 files changed, 69 insertions(+), 22 deletions(-) diff --git a/webapp/src/db.ts b/webapp/src/db.ts index 57a1a5f1dd2..cffea8168ff 100644 --- a/webapp/src/db.ts +++ b/webapp/src/db.ts @@ -91,7 +91,7 @@ export class Table { } if (e.status == 400) { // bad request; likely _rev format was wrong or something similiar - pxt.debug(`table: ${e.name}:${e.message} for ${obj._id}#${obj._rev}`); + console.log(`table: bad request ${e.name}:${e.message} for ${obj._id}#${obj._rev}`); // TODO @darzu: // TODO @darzu: what's the right behavior here? Do we ever expect a 400 in normal operation? // return undefined; } diff --git a/webapp/src/workspaces/cloudsyncworkspace.ts b/webapp/src/workspaces/cloudsyncworkspace.ts index 6244120fc1d..49d1ede43ba 100644 --- a/webapp/src/workspaces/cloudsyncworkspace.ts +++ b/webapp/src/workspaces/cloudsyncworkspace.ts @@ -90,8 +90,10 @@ export function createCachedWorkspace(ws: WorkspaceProvider): CachedWorkspacePro const newProjs = oldProjs for (let id of Object.keys(newProjs)) { const newHdr = newHdrsMap[id] - if (!newHdr || hasChanged(newHdr, oldHdrsMap[id])) + if (!newHdr || hasChanged(newHdr, oldHdrsMap[id])) { + console.log(`DELETE ${id}`) // TODO @darzu: dbg delete newProjs[id] + } } // save results @@ -107,6 +109,7 @@ export function createCachedWorkspace(ws: WorkspaceProvider): CachedWorkspacePro return cacheHdrs } async function getAsync(h: Header): Promise { + // TODO @darzu: should the semantics of this check the header version? await pendingUpdate; if (!cacheProjs[h.id]) { // fetch @@ -118,11 +121,19 @@ export function createCachedWorkspace(ws: WorkspaceProvider): CachedWorkspacePro } async function setAsync(h: Header, prevVer: Version, text?: ScriptText): Promise { await pendingUpdate; + // update cached projects cacheProjs[h.id] = { header: h, text, version: prevVer } + // update headers list + if (!cacheHdrsMap[h.id]) { + cacheHdrs.push(h) + } + // update headers map + cacheHdrsMap[h.id] = h + // update mod time cacheModTime = Math.max(cacheModTime, h.modificationTime) const res = await ws.setAsync(h, prevVer, text) return res; @@ -172,16 +183,20 @@ export function createCloudSyncWorkspace(cloud: WorkspaceProvider, cloudLocal: W const firstCachePull = Promise.all([cloudCache.firstSync(), localCache.firstSync()]) const pendingCacheSync = () => Promise.all([cloudCache.pendingSync(), localCache.pendingSync()]) const getLastModTime = () => Math.max(cloudCache.getLastModTime(), localCache.getLastModTime()) + const needsSync = () => cloudCache.getLastModTime() !== localCache.getLastModTime() + + // TODO @darzu: we could frequently check the last mod times to see if a sync is in order? // TODO @darzu: multi-tab safety for cloudLocal // TODO @darzu: when two workspaces disagree on last mod time, we should sync? - let pendingSync: Promise = synchronizeInternal(); // TODO @darzu: kick this off immediately? + let pendingSync: Promise = synchronizeInternal(); const firstSync = pendingSync; + async function synchronize(expectedLastModTime?:number): Promise { if (pendingSync.isPending()) return pendingSync - pendingSync = synchronizeInternal() + pendingSync = synchronizeInternal(expectedLastModTime) return pendingSync } @@ -197,6 +212,9 @@ export function createCloudSyncWorkspace(cloud: WorkspaceProvider, cloudLocal: W // TODO @darzu: worklist this? // TODO @darzu: track pending saves + // TODO @darzu: dbg + console.log(`transfer ${fromH.id}(${fromH.modificationTime},${fromH._rev}) => (${toH.modificationTime},${toH._rev})`) + const newPrj = await fromWs.getAsync(fromH) // we need the old project if any exists so we know what prevVersion to pass @@ -217,16 +235,48 @@ export function createCloudSyncWorkspace(cloud: WorkspaceProvider, cloudLocal: W return newH; } - async function synchronizeInternal(): Promise { - await pendingCacheSync() - + async function synchronizeInternal(expectedLastModTime?:number): Promise { console.log("cloudsyncworkspace: synchronizeInternal") - if (cloudCache.getLastModTime() === localCache.getLastModTime()) { - // we're synced up ? - return false + // TODO @darzu: error: circular promise resolution chain + + // case 1: everything should be synced up, we're just polling the server + // expectedLastModTime = 0 + // we definitely want cloudCache to synchronize + // we don't need localCache to synchronize + // we want to wait on localCache.pendingSync + // case 2: we suspect localCache is out of date (from other tab changes) + // expectedLastModTime = someValueFromOtherTab + // we don't need cloudCache to synchronize + // we definitely want localCache to synchronize + // we want to wait on cloudCache.pendingSync + // case 3: createCloudSyncWorkspace is first called (first sync) + // expectedLastModTime = 0 + // we don't need cloudCache to synchronize + // we don't need localCache to synchronize + // we want to wait on localCache.pendingSync + // TODO @darzu: need to think through and compare how this would work with git + + // TODO @darzu: not sure what case would hit this: + // if (!expectedLastModTime) { + // // first check if there is a known disagreement before we force each side to deep sync + // if (cloudCache.getLastModTime() !== localCache.getLastModTime()) + // expectedLastModTime = getLastModTime(); + // } + + // wait for each side to sync + // TODO @darzu: this isn't symmetric. Can we fix the abstraction so it is? + if (!expectedLastModTime) { + await Promise.all([cloudCache.synchronize(), localCache.pendingSync()]) + } else { + await Promise.all([cloudCache.pendingSync(), localCache.synchronize(expectedLastModTime)]) } - // TODO @darzu: compare mod times to see if we need to sync? + + // TODO @darzu: mod time isn't sufficient for a set of headers; maybe hash or merkle tree + // // short circuit if there aren't changes + // if (cloudCache.getLastModTime() === localCache.getLastModTime()) { + // return false + // } // TODO @darzu: re-generalize? const left = cloudCache; @@ -236,12 +286,8 @@ export function createCloudSyncWorkspace(cloud: WorkspaceProvider, cloudLocal: W disjointSets: DisjointSetsStrategy.Synchronize } - // wait for each side to sync - await Promise.all([cloudCache.synchronize(), localCache.synchronize()]) - - // TODO @darzu: short circuit if there aren't changes ? - const lHdrsList = await left.listAsync() - const rHdrsList = await right.listAsync() + const lHdrsList = left.listSync() + const rHdrsList = right.listSync() const lHdrs = U.toDictionary(lHdrsList, h => h.id) const rHdrs = U.toDictionary(rHdrsList, h => h.id) @@ -273,7 +319,7 @@ export function createCloudSyncWorkspace(cloud: WorkspaceProvider, cloudLocal: W const rPushPromises = rToPush.map(h => transfer(lHdrs[h.id], rHdrs[h.id], left, right)) // wait - // TODO @darzu: batching? throttling? incremental? + // TODO @darzu: worklist? batching? throttling? incremental? const changed = await Promise.all([...lPushPromises, ...rPushPromises]) // TODO @darzu: what about mod time changes? @@ -290,7 +336,7 @@ export function createCloudSyncWorkspace(cloud: WorkspaceProvider, cloudLocal: W } async function setAsync(h: Header, prevVer: any, text?: ScriptText): Promise { await pendingSync - // TODO @darzu: use a queue to sync to backend + // TODO @darzu: use a queue to sync to backend and make sure this promise is part of the pending sync set const cloudPromise = cloudCache.setAsync(h, prevVer, text) // TODO @darzu: also what to do with the return value ? return await localCache.setAsync(h, prevVer, text) @@ -311,7 +357,7 @@ export function createCloudSyncWorkspace(cloud: WorkspaceProvider, cloudLocal: W // TODO @darzu: debug logging firstSync.then(c => { - console.log("first update:") + console.log("cloudSyncWS first update:") console.dir(localCache.listSync().map(h => ({id: h.id, t: h.modificationTime}))) }) diff --git a/webapp/src/workspaces/jointworkspace.ts b/webapp/src/workspaces/jointworkspace.ts index 47b87911771..590dc9b0f49 100644 --- a/webapp/src/workspaces/jointworkspace.ts +++ b/webapp/src/workspaces/jointworkspace.ts @@ -71,7 +71,7 @@ export function createJointWorkspace(...all: CachedWorkspaceProvider[]): CachedW const getLastModTime = () => Math.max(...all.map(w => w.getLastModTime())) async function synchronize(expectedLastModTime?: number): Promise { - return (await Promise.all(all.map(w => w.synchronize()))) + return (await Promise.all(all.map(w => w.synchronize(expectedLastModTime)))) .reduce((p, n) => p || n, false) } function listSync(): Header[] { diff --git a/webapp/src/workspaces/workspace.ts b/webapp/src/workspaces/workspace.ts index dc66a998e11..ce57399a54a 100644 --- a/webapp/src/workspaces/workspace.ts +++ b/webapp/src/workspaces/workspace.ts @@ -109,7 +109,8 @@ export function setupWorkspace(kind: WorkspaceKind): void { const localChoice = chooseWorkspace(implType); // TODO @darzu: if (auth.loggedInSync()) { - console.log("logged in") // TODO @darzu: + console.log("logged in") // TODO @darzu: dbg + // TODO @darzu: need per-user cloud-local const cloudApis = cloudWorkspace.provider const localCloud = createBrowserDbWorkspace("cloud-local"); // TODO @darzu: use user choice for this too? // const localCloud = createBrowserDbWorkspace(""); // TODO @darzu: undo dbg From d711126b72818bbf149b353911521a1b9f61c586 Mon Sep 17 00:00:00 2001 From: darzu Date: Fri, 18 Dec 2020 20:47:43 -0800 Subject: [PATCH 37/52] fix debug crash --- webapp/src/workspaces/cloudsyncworkspace.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp/src/workspaces/cloudsyncworkspace.ts b/webapp/src/workspaces/cloudsyncworkspace.ts index 49d1ede43ba..754c480367d 100644 --- a/webapp/src/workspaces/cloudsyncworkspace.ts +++ b/webapp/src/workspaces/cloudsyncworkspace.ts @@ -213,7 +213,7 @@ export function createCloudSyncWorkspace(cloud: WorkspaceProvider, cloudLocal: W // TODO @darzu: track pending saves // TODO @darzu: dbg - console.log(`transfer ${fromH.id}(${fromH.modificationTime},${fromH._rev}) => (${toH.modificationTime},${toH._rev})`) + console.log(`transfer ${fromH.id}(${fromH.modificationTime},${fromH._rev}) => (${toH?.modificationTime},${toH?._rev})`) const newPrj = await fromWs.getAsync(fromH) From 71941fdcaf6e6955303b081291ffd016ea48a5a8 Mon Sep 17 00:00:00 2001 From: darzu Date: Mon, 21 Dec 2020 14:19:06 -0800 Subject: [PATCH 38/52] debugging --- webapp/src/workspaces/browserdbworkspace.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/webapp/src/workspaces/browserdbworkspace.ts b/webapp/src/workspaces/browserdbworkspace.ts index aca70812dfb..45c33e552e2 100644 --- a/webapp/src/workspaces/browserdbworkspace.ts +++ b/webapp/src/workspaces/browserdbworkspace.ts @@ -48,6 +48,7 @@ export function createBrowserDbWorkspace(namespace: string): BrowserDbWorkspaceP async function setAsync(h: Header, prevVer: any, text?: ScriptText): Promise { // TODO @darzu: debug logging if (!text) { + // TODO @darzu: trace down why... this is a real bug console.log("!!! setAsync without text :(") // console.dir(h) } else { @@ -66,8 +67,12 @@ export function createBrowserDbWorkspace(namespace: string): BrowserDbWorkspaceP textVer = await textDb.setAsync(textEnt) } catch (e) {} - if (!textVer) + if (!textVer) { console.log(`! failed to set text for id:${h.id},pv:${prevVer}`); // TODO @darzu: dbg logging + const oldTxt = await textDb.getAsync(h.id) + console.dir(`! text ${h.id} actually is: ${oldTxt._rev}`) + console.dir(oldTxt) + } let hdrVer: string; try { From ae417782c6748f9a6780927fab80868a468afad0 Mon Sep 17 00:00:00 2001 From: darzu Date: Wed, 23 Dec 2020 16:47:49 -0800 Subject: [PATCH 39/52] big progress on race conditions & sanity --- pxteditor/workspace.ts | 4 ++ webapp/src/app.tsx | 5 +- webapp/src/db.ts | 14 +++-- webapp/src/workspaces/browserdbworkspace.ts | 8 ++- webapp/src/workspaces/browserworkspace.ts | 6 +- webapp/src/workspaces/cloudsyncworkspace.ts | 22 ++++++-- webapp/src/workspaces/idbworkspace.ts | 11 +++- webapp/src/workspaces/jointworkspace.ts | 16 +++--- webapp/src/workspaces/workspace.ts | 62 +++++++++++++++++++-- 9 files changed, 120 insertions(+), 28 deletions(-) diff --git a/pxteditor/workspace.ts b/pxteditor/workspace.ts index 3de8ae09ad4..48eab71cbed 100644 --- a/pxteditor/workspace.ts +++ b/pxteditor/workspace.ts @@ -30,6 +30,10 @@ namespace pxt.workspace { export interface WorkspaceProvider { listAsync(): Promise; // called from workspace.syncAsync (including upon startup) + /* + Tries to get the corrisponding File with the current version if it exists. + If it does not exist, returns undefined. + */ getAsync(h: Header): Promise; setAsync(h: Header, prevVersion: Version, text?: ScriptText): Promise; deleteAsync?: (h: Header, prevVersion: Version) => Promise; diff --git a/webapp/src/app.tsx b/webapp/src/app.tsx index dff9412966a..d858c23a800 100644 --- a/webapp/src/app.tsx +++ b/webapp/src/app.tsx @@ -4493,6 +4493,7 @@ document.addEventListener("DOMContentLoaded", () => { const isSandbox = pxt.shell.isSandboxMode() || pxt.shell.isReadOnly(); const isController = pxt.shell.isControllerMode(); const theme = pxt.appTarget.appTheme; + // TODO @darzu: this is bad. we shouldn't be choosing a workspace in two places (see "chooseWorkspace") if (query["ws"]) { workspace.setupWorkspace(query["ws"] as workspace.WorkspaceKind) } @@ -4502,7 +4503,9 @@ document.addEventListener("DOMContentLoaded", () => { else if (pxt.BrowserUtils.isIpcRenderer()) workspace.setupWorkspace("idb"); // TODO @darzu: uncomment. this disables filesystem workspace //else if (pxt.BrowserUtils.isLocalHost() || pxt.BrowserUtils.isPxtElectron()) workspace.setupWorkspace("fs"); - else workspace.setupWorkspace("browser"); + else { + workspace.setupWorkspace("browser"); + } Promise.resolve() .then(async () => { const href = window.location.href; diff --git a/webapp/src/db.ts b/webapp/src/db.ts index cffea8168ff..b67be4489ef 100644 --- a/webapp/src/db.ts +++ b/webapp/src/db.ts @@ -51,10 +51,16 @@ export class Table { constructor(public name: string) { } getAsync(id: string): Promise { - return getDbAsync().then(db => db.get(this.name + "--" + id)).then((v: any) => { - v.id = id - return v - }) + return getDbAsync().then(db => db.get(this.name + "--" + id)) + .then((v: any) => { + v.id = id + return v + }) + .catch(e => { + // not found + // TODO @darzu: trace users to see if this new behavior breaks assumptions + return undefined + }) } getAllAsync(): Promise { diff --git a/webapp/src/workspaces/browserdbworkspace.ts b/webapp/src/workspaces/browserdbworkspace.ts index 45c33e552e2..339200383d9 100644 --- a/webapp/src/workspaces/browserdbworkspace.ts +++ b/webapp/src/workspaces/browserdbworkspace.ts @@ -16,6 +16,10 @@ export interface BrowserDbWorkspaceProvider extends pxt.workspace.WorkspaceProvi // TODO @darzu: very important for _rev and _id export function createBrowserDbWorkspace(namespace: string): BrowserDbWorkspaceProvider { + if (!namespace) { + console.log("BAD default namespace created") + console.trace(); + } const prefix = namespace ? namespace + "-" : "" const headerDb = new db.Table(`${prefix}header`); const textDb = new db.Table(`${prefix}text`); @@ -39,6 +43,8 @@ export function createBrowserDbWorkspace(namespace: string): BrowserDbWorkspaceP } async function getAsync(h: Header): Promise { const resp: TextDbEntry = await textDb.getAsync(h.id) + if (!resp) + return undefined; return { header: h, text: resp.files, @@ -52,7 +58,7 @@ export function createBrowserDbWorkspace(namespace: string): BrowserDbWorkspaceP console.log("!!! setAsync without text :(") // console.dir(h) } else { - console.log(`setAsync ${namespace || "def"}:(${h.id}, ${h.modificationTime}, ${prevVer}) :)`) + console.log(`setAsync ${namespace || "default"}:(${h.id}, ${h.modificationTime}, ${prevVer}) :)`) } const textEnt: TextDbEntry = { diff --git a/webapp/src/workspaces/browserworkspace.ts b/webapp/src/workspaces/browserworkspace.ts index 4284bc8d88b..58abb3a5c32 100644 --- a/webapp/src/workspaces/browserworkspace.ts +++ b/webapp/src/workspaces/browserworkspace.ts @@ -38,6 +38,7 @@ const getPreviousDbPrefix = () => { } async function createAndMigrateBrowserDb(): Promise { + console.log("BAD createAndMigrateBrowserDb") // TODO @darzu: trace const currentDbPrefix = getCurrentDbPrefix(); let currDb: BrowserDbWorkspaceProvider; if (currentDbPrefix) { @@ -69,6 +70,7 @@ async function createAndMigrateBrowserDb(): Promise } export async function copyProjectToLegacyEditor(h: Header, majorVersion: number): Promise
{ + console.log("BAD copyProjectToLegacyEditor") // TODO @darzu: trace await init(); const prefix = getVersionedDbPrefix(majorVersion); @@ -86,7 +88,8 @@ export async function copyProjectToLegacyEditor(h: Header, majorVersion: number) } // TODO @darzu: might be a better way to provide this wrapping and handle the migration -export const provider: WorkspaceProvider = { +// TODO @darzu: export +const provider: WorkspaceProvider = { listAsync: async () => { await init(); return currentDb.listAsync(); @@ -97,6 +100,7 @@ export const provider: WorkspaceProvider = { }, setAsync: async (h: Header, prevVersion: pxt.workspace.Version, text?: ScriptText) => { await init(); + console.log("BAD setAsync") // TODO @darzu: tracing usage return currentDb.setAsync(h, prevVersion, text); }, deleteAsync: async (h: Header, prevVersion: pxt.workspace.Version) => { diff --git a/webapp/src/workspaces/cloudsyncworkspace.ts b/webapp/src/workspaces/cloudsyncworkspace.ts index 754c480367d..fe0fe334263 100644 --- a/webapp/src/workspaces/cloudsyncworkspace.ts +++ b/webapp/src/workspaces/cloudsyncworkspace.ts @@ -3,7 +3,7 @@ import U = pxt.Util; type Header = pxt.workspace.Header; type ScriptText = pxt.workspace.ScriptText; -type F = pxt.workspace.File; +type WsFile = pxt.workspace.File; type Version = pxt.workspace.Version; type WorkspaceProvider = pxt.workspace.WorkspaceProvider; @@ -20,7 +20,7 @@ export interface CachedWorkspaceProvider extends WorkspaceProvider { firstSync(): Promise, listSync(): Header[], hasSync(h: Header): boolean, - tryGetSync(h: Header): F + tryGetSync(h: Header): WsFile } function computeLastModTime(hdrs: Header[]): number { @@ -35,7 +35,7 @@ function hasChanged(a: Header, b: Header): boolean { export function createCachedWorkspace(ws: WorkspaceProvider): CachedWorkspaceProvider { let cacheHdrs: Header[] = [] let cacheHdrsMap: {[id: string]: Header} = {}; - let cacheProjs: {[id: string]: F} = {}; + let cacheProjs: {[id: string]: WsFile} = {}; let cacheModTime: number = 0; @@ -108,7 +108,7 @@ export function createCachedWorkspace(ws: WorkspaceProvider): CachedWorkspacePro await pendingUpdate; return cacheHdrs } - async function getAsync(h: Header): Promise { + async function getAsync(h: Header): Promise { // TODO @darzu: should the semantics of this check the header version? await pendingUpdate; if (!cacheProjs[h.id]) { @@ -136,6 +136,17 @@ export function createCachedWorkspace(ws: WorkspaceProvider): CachedWorkspacePro // update mod time cacheModTime = Math.max(cacheModTime, h.modificationTime) const res = await ws.setAsync(h, prevVer, text) + if (res) { + // update cached projec + cacheProjs[h.id] = { + header: h, + text, + version: res + } + } else { + // conflict; delete cache + delete cacheProjs[h.id] + } return res; } async function deleteAsync(h: Header, prevVer: Version): Promise { @@ -330,12 +341,13 @@ export function createCloudSyncWorkspace(cloud: WorkspaceProvider, cloudLocal: W await pendingSync return localCache.listAsync() } - async function getAsync(h: Header): Promise { + async function getAsync(h: Header): Promise { await pendingSync return localCache.getAsync(h) } async function setAsync(h: Header, prevVer: any, text?: ScriptText): Promise { await pendingSync + // TODO @darzu: cannot pass prevVer to both of these.. they have different meanings on the different platforms // TODO @darzu: use a queue to sync to backend and make sure this promise is part of the pending sync set const cloudPromise = cloudCache.setAsync(h, prevVer, text) // TODO @darzu: also what to do with the return value ? diff --git a/webapp/src/workspaces/idbworkspace.ts b/webapp/src/workspaces/idbworkspace.ts index 1890063fc15..74ebd9e8026 100644 --- a/webapp/src/workspaces/idbworkspace.ts +++ b/webapp/src/workspaces/idbworkspace.ts @@ -21,6 +21,7 @@ const KEYPATH = "id"; // This function migrates existing projectes in pouchDb to indexDb // From browserworkspace to idbworkspace async function migrateBrowserWorkspaceAsync(): Promise { + console.log("BAD migrateBrowserWorkspaceAsync called") // TODO @darzu: this shouldn't be needed const db = await getDbAsync(); const allDbHeaders = await db.getAllAsync(HEADERS_TABLE); if (allDbHeaders.length) { @@ -28,8 +29,10 @@ async function migrateBrowserWorkspaceAsync(): Promise { return; } + const ws: WorkspaceProvider = null; // TODO @darzu: browserworkspace.provider + const copyProject = async (h: pxt.workspace.Header): Promise => { - const resp = await browserworkspace.provider.getAsync(h); + const resp = await ws.getAsync(h); // Ignore metadata of the previous script so they get re-generated for the new copy delete (resp as any)._id; @@ -38,7 +41,7 @@ async function migrateBrowserWorkspaceAsync(): Promise { await setAsync(h, undefined, resp.text); }; - const previousHeaders = await browserworkspace.provider.listAsync(); + const previousHeaders = await ws.listAsync(); await Promise.all(previousHeaders.map(h => copyProject(h))); } @@ -75,7 +78,9 @@ async function getDbAsync(): Promise { } async function listAsync(): Promise { - await migrateBrowserWorkspaceAsync(); + // TODO @darzu: + console.log("idbworkspace:listAsync") + // await migrateBrowserWorkspaceAsync(); const db = await getDbAsync(); return db.getAllAsync(HEADERS_TABLE); } diff --git a/webapp/src/workspaces/jointworkspace.ts b/webapp/src/workspaces/jointworkspace.ts index 590dc9b0f49..0ad37e45114 100644 --- a/webapp/src/workspaces/jointworkspace.ts +++ b/webapp/src/workspaces/jointworkspace.ts @@ -86,24 +86,26 @@ export function createJointWorkspace(...all: CachedWorkspaceProvider[]): CachedW .reduce((p, n) => [...p, ...n], []) } function getWorkspaceFor(h: Header): CachedWorkspaceProvider { - return all.reduce((p, n) => p || n.hasSync(h) ? n : p, null) + return all.reduce((p, n) => p || (n.hasSync(h) ? n : null), null) } async function getAsync(h: Header): Promise { await pendingSync() - // chose the first matching one - const ws = getWorkspaceFor(h) ?? all[0] - return ws.getAsync(h) + // choose the first matching one + const ws = getWorkspaceFor(h) + return ws?.getAsync(h) ?? undefined } function tryGetSync(h: Header): File { - // chose the first matching one - const ws = getWorkspaceFor(h) ?? all[0] - return ws.tryGetSync(h) + // choose the first matching one + const ws = getWorkspaceFor(h) + return ws?.tryGetSync(h) ?? undefined } function hasSync(h: Header): boolean { return all.reduce((p, n) => p || n.hasSync(h), false) } async function setAsync(h: Header, prevVer: any, text?: ScriptText): Promise { await pendingSync() + console.log("joint:setAsync") + console.dir(all.map(w => w.hasSync(h))) const ws = getWorkspaceFor(h) ?? all[0] return ws.setAsync(h, prevVer, text) } diff --git a/webapp/src/workspaces/workspace.ts b/webapp/src/workspaces/workspace.ts index ce57399a54a..17d3eebda34 100644 --- a/webapp/src/workspaces/workspace.ts +++ b/webapp/src/workspaces/workspace.ts @@ -95,7 +95,10 @@ function chooseWorkspace(kind: WorkspaceKind = "browser"): pxt.workspace.Workspa return cloudWorkspace.provider; case "browser": default: - return browserworkspace.provider + // TODO @darzu: + console.log("chooseWorkspace browser") + return createBrowserDbWorkspace("") + // return browserworkspace.provider } } @@ -363,8 +366,10 @@ function checkHeaderSession(h: Header): void { export function initAsync() { if (!impl) { // TODO @darzu: hmmmm we should be use setupWorkspace - impl = createCachedWorkspace(browserworkspace.provider); - implType = "browser"; + console.log("BAD init browser workspace") // TODO @darzu: + // TODO @darzu: + // impl = createCachedWorkspace(browserworkspace.provider); + // implType = "browser"; } return syncAsync() @@ -460,7 +465,48 @@ export function forceSaveAsync(h: Header, text?: ScriptText, isCloud?: boolean): return saveAsync(h, text, isCloud); } -export function saveAsync(h: Header, text?: ScriptText, isCloud?: boolean): Promise { +export async function saveAsync(header: Header, text?: ScriptText, isCloud?: boolean): Promise { + console.log(`workspace:saveAsync ${header.id}`) + // TODO @darzu: port over from saveAsync2 + let prj = await impl.getAsync(header) // TODO @darzu: dbg + if (!prj) { + // first save + console.log(`first save: ${header.id}`) // TODO @darzu: dbg + prj = { + header, + text, + version: null + } + } else { + if (prj.text === text) { + // we're getting a double save.. no point + console.log("BAD double save!") // TODO @darzu: dbg + // TODO @darzu: + // return + } + } + + try { + const res = await impl.setAsync(header, prj.version, text); + if (!res) { + // conflict occured + console.log(`conflict occured for ${header.id} at ${prj.version}`) // TODO @darzu: dbg + // TODO @darzu: what to do? probably nothing + } + } catch (e) { + // Write failed; use in memory db. + // TODO @darzu: POLICY + console.log("switchToMemoryWorkspace (1)") // TODO @darzu: + console.dir(e) + // await switchToMemoryWorkspace("write failed"); + // await impl.setAsync(header, prj.version, text); + } + + return; +} + +export function saveAsync2(h: Header, text?: ScriptText, isCloud?: boolean): Promise { + // TODO @darzu: rebuild this pxt.debug(`workspace: save ${h.id}`) if (h.isDeleted) clearHeaderSession(h); @@ -563,7 +609,7 @@ function computePath(h: Header) { export function importAsync(h: Header, text: ScriptText, isCloud = false) { // TODO @darzu: why does import bypass workspaces or does it? - console.log(`importAsync: ${h.id}`); + console.log(`importAsync: ${h.id}`); // TODO @darzu: dbg h.path = computePath(h) const e: File = { header: h, @@ -575,6 +621,7 @@ export function importAsync(h: Header, text: ScriptText, isCloud = false) { } export function installAsync(h0: InstallHeader, text: ScriptText) { + console.log(`workspace:installAsync ${h0.pubId}`) // TODO @darzu: dbg // TODO @darzu: why do we "install" here? how does that relate to "import"? This is 5 years old... U.assert(h0.target == pxt.appTarget.id); @@ -1588,7 +1635,10 @@ export function listAssetsAsync(id: string): Promise { } export function isBrowserWorkspace() { - return impl === browserworkspace.provider; + // TODO @darzu: trace all uses. this shouldn't be needed + console.log("workspace.ts:isBrowserWorkspace") + return false + // return impl === browserworkspace.provider; } export function fireEvent(ev: pxt.editor.events.Event) { From be015c5fdbdea71579980fb1f808ab99f13ec3a7 Mon Sep 17 00:00:00 2001 From: darzu Date: Wed, 23 Dec 2020 20:01:42 -0800 Subject: [PATCH 40/52] deprecate allscripts --- webapp/src/workspaces/workspace.ts | 57 +++++++++++++++++++++--------- 1 file changed, 40 insertions(+), 17 deletions(-) diff --git a/webapp/src/workspaces/workspace.ts b/webapp/src/workspaces/workspace.ts index 17d3eebda34..4510c834a1f 100644 --- a/webapp/src/workspaces/workspace.ts +++ b/webapp/src/workspaces/workspace.ts @@ -37,21 +37,37 @@ type WorkspaceProvider = pxt.workspace.WorkspaceProvider; type InstallHeader = pxt.workspace.InstallHeader; type File = pxt.workspace.File; +// TODO @darzu: todo list: +// [ ] remove / fix header session methods +// [ ] remove forceSaveAsync +// [x] remove allScripts +// [ ] remove headerQ +// [ ] invalidate virtual api: + // data.invalidateHeader("header", hd); + // data.invalidateHeader("text", hd); + // data.invalidateHeader("pkg-git-status", hd); + // data.invalidate("gh-commits:*"); // invalidate commits just in case +// [ ] remove syncAsync + + // TODO @darzu: remove. redudant w/ implCache -let allScripts: File[] = []; +// let allScripts: File[] = []; // TODO @darzu: del let headerQ = new U.PromiseQueue(); let impl: CachedWorkspaceProvider; -// TODO @darzu: +// TODO @darzu: del // let implCache: CachedWorkspaceProvider; let implType: WorkspaceKind; +// TODO @darzu: del function lookup(id: string): File { - // TODO @darzu: - // const hdr = implCache.listSync().find(x => x.header.id == id || x.header.path == id) + // TODO @darzu: what is .path used for? + const hdr = impl.listSync().find(h => h.id == id || h.path == id) + return impl.tryGetSync(hdr) + // TODO @darzu: del // implCache.getSync(); - return allScripts.find(x => x.header.id == id || x.header.path == id); + // return allScripts.find(x => x.header.id == id || x.header.path == id); } export function gitsha(data: string, encoding: "utf-8" | "base64" = "utf-8") { @@ -293,7 +309,9 @@ export function restoreFromBackupAsync(h: Header) { } function cleanupBackupsAsync() { - const allHeaders = allScripts.map(e => e.header); + const allHeaders = impl.listSync(); + // TODO @darzu: del + // const allHeaders = allScripts.map(e => e.header); const refMap: pxt.Map = {}; // Figure out which scripts have backups @@ -328,8 +346,9 @@ function maybeSyncHeadersAsync(): Promise { function refreshHeadersSession() { // TODO @darzu: carefully handle this // use # of scripts + time of last mod as key - sessionID = allScripts.length + ' ' + allScripts - .map(h => h.header.modificationTime) + const hdrs = impl.listSync() + sessionID = hdrs.length + ' ' + hdrs + .map(h => h.modificationTime) .reduce((l, r) => Math.max(l, r), 0) .toString() if (isHeadersSessionOutdated()) { @@ -541,9 +560,10 @@ export function saveAsync2(h: Header, text?: ScriptText, isCloud?: boolean): Pro // perma-delete if (h.isDeleted && h.blobVersion_ == "DELETED") { // TODO @darzu: "isDelete" is a command flag????? argh.. - let idx = allScripts.indexOf(e) - U.assert(idx >= 0) - allScripts.splice(idx, 1) + // TODO @darzu: del: + // let idx = allScripts.indexOf(e) + // U.assert(idx >= 0) + // allScripts.splice(idx, 1) return headerQ.enqueue(h.id, () => fixupVersionAsync(e).then(() => impl.deleteAsync ? impl.deleteAsync(h, e.version) : impl.setAsync(h, e.version, {}))) @@ -594,6 +614,7 @@ export function saveAsync2(h: Header, text?: ScriptText, isCloud?: boolean): Pro } function computePath(h: Header) { + // TODO @darzu: what's the deal with this path? let path = h.name.replace(/[^a-zA-Z0-9]+/g, " ").trim().replace(/ /g, "-") if (!path) path = "Untitled"; // do not translate @@ -616,7 +637,8 @@ export function importAsync(h: Header, text: ScriptText, isCloud = false) { text: text, version: null } - allScripts.push(e) + // TODO @darzu: del + // allScripts.push(e) return forceSaveAsync(h, text, isCloud) } @@ -674,7 +696,7 @@ export function duplicateAsync(h: Header, text: ScriptText, newName?: string): P export function createDuplicateName(h: Header) { let reducedName = h.name.indexOf("#") > -1 ? h.name.substring(0, h.name.lastIndexOf('#')).trim() : h.name; - let names = U.toDictionary(allScripts.filter(e => !e.header.isDeleted), e => e.header.name) + let names = U.toDictionary(impl.listSync().filter(h => !h.isDeleted), h => h.name) let n = 2 while (names.hasOwnProperty(reducedName + " #" + n)) n++ @@ -1551,10 +1573,10 @@ export function syncAsync(): Promise { .then(() => impl.listAsync()); }) .then(headers => { - const existing = U.toDictionary(allScripts || [], h => h.header.id) // this is an in-place update the header instances - allScripts = headers.map(hd => { - let ex = existing[hd.id] + // TODO @darzu: del "let" + let allScripts = headers.map(hd => { + let ex = impl.tryGetSync(hd) if (ex) { if (JSON.stringify(ex.header) !== JSON.stringify(hd)) { U.jsonCopyFrom(ex.header, hd) @@ -1591,7 +1613,8 @@ export function syncAsync(): Promise { export async function resetAsync() { // TODO @darzu: this should just pass through to workspace impl console.log("resetAsync (1)") // TODO @darzu: - allScripts = [] + // TODO @darzu: del + // allScripts = [] await impl.resetAsync(); console.log("resetAsync (2)") // TODO @darzu: await cloudsync.resetAsync(); From e83fd6e55a9f443ef3a4239e36995eadb82474c6 Mon Sep 17 00:00:00 2001 From: darzu Date: Mon, 28 Dec 2020 11:06:06 -0800 Subject: [PATCH 41/52] investigative comments and todos --- webapp/src/app.tsx | 8 +++++- webapp/src/workspaces/browserdbworkspace.ts | 4 +++ webapp/src/workspaces/cloudsyncworkspace.ts | 18 +++++++++++-- webapp/src/workspaces/workspace.ts | 30 ++++++++++++++++----- 4 files changed, 50 insertions(+), 10 deletions(-) diff --git a/webapp/src/app.tsx b/webapp/src/app.tsx index d858c23a800..f07af9484bf 100644 --- a/webapp/src/app.tsx +++ b/webapp/src/app.tsx @@ -263,8 +263,13 @@ export class ProjectView } this.saveFileAsync().done(); } else if (active) { + // TODO @darzu: new code path: + // workspace.syncAsync().then(changed => this.reloadAsync()) + // reloadAsync: this.loadHeaderAsync() + // OR: subscribe to data api, on change, reload + data.invalidate("header:*") - if (workspace.isHeadersSessionOutdated() + if (workspace.isHeadersSessionOutdated() // TODO @darzu: sync check point || workspace.isHeaderSessionOutdated(this.state.header)) { pxt.debug('workspace: changed, reloading...') let id = this.state.header ? this.state.header.id : ''; @@ -1299,6 +1304,7 @@ export class ProjectView return checkAsync.then(() => this.openHome()); let p = Promise.resolve(); + // TODO @darzu: sync checkpoint if (workspace.isHeadersSessionOutdated()) { // reload header before loading pxt.log(`sync before load`) p = p.then(() => workspace.syncAsync().then(() => { })) diff --git a/webapp/src/workspaces/browserdbworkspace.ts b/webapp/src/workspaces/browserdbworkspace.ts index 339200383d9..93365a2097c 100644 --- a/webapp/src/workspaces/browserdbworkspace.ts +++ b/webapp/src/workspaces/browserdbworkspace.ts @@ -24,6 +24,10 @@ export function createBrowserDbWorkspace(namespace: string): BrowserDbWorkspaceP const headerDb = new db.Table(`${prefix}header`); const textDb = new db.Table(`${prefix}text`); + // TODO @darzu: dz: + // return pxt.storage.getLocal('workspacesessionid') != sessionID; + // pxt.storage.setLocal('workspacesessionid', sessionID); + // TODO @darzu: const printDbg = async () => { const hdrs: pxt.workspace.Header[] = await headerDb.getAllAsync(); diff --git a/webapp/src/workspaces/cloudsyncworkspace.ts b/webapp/src/workspaces/cloudsyncworkspace.ts index fe0fe334263..ad837213b5b 100644 --- a/webapp/src/workspaces/cloudsyncworkspace.ts +++ b/webapp/src/workspaces/cloudsyncworkspace.ts @@ -11,11 +11,11 @@ type WorkspaceProvider = pxt.workspace.WorkspaceProvider; // [ ] cache invalidation via header sessions // [ ] enforce soft-delete // pouchdb uses _delete for soft delete -// [ ] need to think more about error handling and retries +// [ ] error handling: conflicts returned as "undefined"; other errors propegate as exceptions export interface CachedWorkspaceProvider extends WorkspaceProvider { getLastModTime(): number, - synchronize(expectedLastModTime?: number): Promise, + synchronize(expectedLastModTime?: number): Promise, // TODO @darzu: name syncAsync? pendingSync(): Promise, firstSync(): Promise, listSync(): Header[], @@ -23,6 +23,12 @@ export interface CachedWorkspaceProvider extends WorkspaceProvider { tryGetSync(h: Header): WsFile } +// TODO @darzu: \/ \/ \/ thread through +export interface SynchronizationReason { + pollCloud?: boolean, + pollSession?: boolean, +} + function computeLastModTime(hdrs: Header[]): number { return hdrs.reduce((p, n) => Math.max(n.modificationTime, p), 0) } @@ -37,6 +43,14 @@ export function createCachedWorkspace(ws: WorkspaceProvider): CachedWorkspacePro let cacheHdrsMap: {[id: string]: Header} = {}; let cacheProjs: {[id: string]: WsFile} = {}; + // TODO @darzu: thinking through workspace sessions + // per header locks? + // for all headers? + // const workspaceID: string = pxt.Util.guidGen(); + // pxt.storage.setLocal('workspaceheadersessionid:' + h.id, workspaceID); + // pxt.storage.removeLocal('workspaceheadersessionid:' + h.id); + // const sid = pxt.storage.getLocal('workspaceheadersessionid:' + h.id); + let cacheModTime: number = 0; function getLastModTime(): number { diff --git a/webapp/src/workspaces/workspace.ts b/webapp/src/workspaces/workspace.ts index 4510c834a1f..eba177a2fe6 100644 --- a/webapp/src/workspaces/workspace.ts +++ b/webapp/src/workspaces/workspace.ts @@ -48,6 +48,7 @@ type File = pxt.workspace.File; // data.invalidateHeader("pkg-git-status", hd); // data.invalidate("gh-commits:*"); // invalidate commits just in case // [ ] remove syncAsync +// [ ] ensure we don't regress https://github.com/microsoft/pxt/issues/7520 // TODO @darzu: remove. redudant w/ implCache @@ -340,17 +341,21 @@ export function isHeadersSessionOutdated() { } function maybeSyncHeadersAsync(): Promise { if (isHeadersSessionOutdated()) // another tab took control - return syncAsync().then(() => { }) + return syncAsync().then(() => { }) // take back control return Promise.resolve(); } -function refreshHeadersSession() { - // TODO @darzu: carefully handle this - // use # of scripts + time of last mod as key +function computeHeadersSessionID() { const hdrs = impl.listSync() - sessionID = hdrs.length + ' ' + hdrs + return hdrs.length + ' ' + hdrs .map(h => h.modificationTime) .reduce((l, r) => Math.max(l, r), 0) .toString() +} +function refreshHeadersSession() { + // TODO @darzu: carefully handle this. + // use # of scripts + time of last mod as key + sessionID = computeHeadersSessionID(); + if (isHeadersSessionOutdated()) { pxt.storage.setLocal('workspacesessionid', sessionID); pxt.debug(`workspace: refreshed headers session to ${sessionID}`); @@ -882,6 +887,7 @@ const BLOCKSDIFF_PREVIEW_PATH = ".github/makecode/blocksdiff.png"; const BINARY_JS_PATH = "assets/js/binary.js"; const VERSION_TXT_PATH = "assets/version.txt"; export async function commitAsync(hd: Header, options: CommitOptions = {}) { + // TODO @darzu: learn how this works await cloudsync.ensureGitHubTokenAsync(); let files = await getTextAsync(hd.id) @@ -1555,10 +1561,20 @@ export async function saveToCloudAsync(h: Header) { // return cloudsync.saveToCloudAsync(h) } + +export async function syncAsync(): Promise { + // TODO @darzu: clean up naming, layering + // TODO @darzu: handle: + // filters?: pxt.editor.ProjectFilters; + // searchBar?: boolean; + return syncAsync2() +} + + // this promise is set while a sync is in progress // cleared when sync is done. let syncAsyncPromise: Promise; -export function syncAsync(): Promise { +export function syncAsync2(): Promise { // TODO @darzu: this function shouldn't be needed ideally pxt.debug("workspace: sync") if (syncAsyncPromise) return syncAsyncPromise; @@ -1584,7 +1600,7 @@ export function syncAsync(): Promise { ex.text = undefined ex.version = undefined // TODO @darzu: handle data API subscriptions on header changed - console.log(`INVALIDATIN header ${hd.id}`) // TODO @darzu: + console.log(`INVALIDATING header ${hd.id}`) // TODO @darzu: data.invalidateHeader("header", hd); data.invalidateHeader("text", hd); data.invalidateHeader("pkg-git-status", hd); From 8700481a218a8745cbb55b4f974cd64273d4083b Mon Sep 17 00:00:00 2001 From: darzu Date: Mon, 28 Dec 2020 20:51:12 -0800 Subject: [PATCH 42/52] reworked headers hash (formerly last-mod-time) --- webapp/src/workspaces/cloudsyncworkspace.ts | 102 +++++++++++--------- webapp/src/workspaces/jointworkspace.ts | 14 +-- webapp/src/workspaces/workspace.ts | 31 +++--- 3 files changed, 78 insertions(+), 69 deletions(-) diff --git a/webapp/src/workspaces/cloudsyncworkspace.ts b/webapp/src/workspaces/cloudsyncworkspace.ts index ad837213b5b..62808069d94 100644 --- a/webapp/src/workspaces/cloudsyncworkspace.ts +++ b/webapp/src/workspaces/cloudsyncworkspace.ts @@ -14,8 +14,8 @@ type WorkspaceProvider = pxt.workspace.WorkspaceProvider; // [ ] error handling: conflicts returned as "undefined"; other errors propegate as exceptions export interface CachedWorkspaceProvider extends WorkspaceProvider { - getLastModTime(): number, - synchronize(expectedLastModTime?: number): Promise, // TODO @darzu: name syncAsync? + getHeadersHash(): string, + synchronize(reason: SynchronizationReason): Promise, // TODO @darzu: name syncAsync? pendingSync(): Promise, firstSync(): Promise, listSync(): Header[], @@ -25,13 +25,19 @@ export interface CachedWorkspaceProvider extends WorkspaceProvider { // TODO @darzu: \/ \/ \/ thread through export interface SynchronizationReason { - pollCloud?: boolean, - pollSession?: boolean, + expectedHeadersHash?: string, + pollStorage?: boolean, } -function computeLastModTime(hdrs: Header[]): number { - return hdrs.reduce((p, n) => Math.max(n.modificationTime, p), 0) +function computeHeadersHash(hdrs: Header[]): string { + // TODO @darzu: should we just do an actual hash? + // [ ] measure perf difference + // [ ] maybe there are some fields we want to ignore; if so, these should likely be moved out of Header interface + return hdrs.length + ' ' + hdrs // TODO @darzu: [ ] use the length component in the workspace internals? + .map(h => h.modificationTime) + .reduce((l, r) => Math.max(l, r), 0) } + function hasChanged(a: Header, b: Header): boolean { // TODO @darzu: use e-tag, _rev, or version uuid instead? return a.modificationTime !== b.modificationTime @@ -51,19 +57,18 @@ export function createCachedWorkspace(ws: WorkspaceProvider): CachedWorkspacePro // pxt.storage.removeLocal('workspaceheadersessionid:' + h.id); // const sid = pxt.storage.getLocal('workspaceheadersessionid:' + h.id); - let cacheModTime: number = 0; - - function getLastModTime(): number { - return cacheModTime; + let cacheHdrsHash: string = ""; + function getHeadersHash(): string { + return cacheHdrsHash; } // TODO @darzu: do we want to kick off the first sync at construction? Side-effects at construction are usually bad.. - let pendingUpdate: Promise = synchronizeInternal(); + let pendingUpdate: Promise = synchronizeInternal({ pollStorage: true }); const firstUpdate = pendingUpdate; - async function synchronize(otherLastModTime?:number): Promise { + async function synchronize(reason: SynchronizationReason): Promise { if (pendingUpdate.isPending()) return pendingUpdate - pendingUpdate = synchronizeInternal(otherLastModTime) + pendingUpdate = synchronizeInternal(reason) return pendingUpdate } @@ -72,25 +77,28 @@ export function createCachedWorkspace(ws: WorkspaceProvider): CachedWorkspacePro cacheHdrs = [] cacheHdrsMap = {} cacheProjs = {} - cacheModTime = 0 + cacheHdrsHash = "" } - async function synchronizeInternal(expectedLastModTime?:number): Promise { + async function synchronizeInternal(reason: SynchronizationReason): Promise { // remember our old cache, we might keep items from it later const oldHdrs = cacheHdrs const oldHdrsMap = cacheHdrsMap const oldProjs = cacheProjs - const oldModTime = cacheModTime + const oldHdrsHash = cacheHdrsHash - if (expectedLastModTime && expectedLastModTime !== cacheModTime) { - // we've been told to invalidate, but we don't know specific - // headers yet so do the conservative thing and reset all + const hashDesync = reason.expectedHeadersHash && reason.expectedHeadersHash !== cacheHdrsHash + const needSync = !cacheHdrsHash || hashDesync || reason.pollStorage; + if (hashDesync) { + // TODO @darzu: does this buy us anything? eraseCache() + } else if (!needSync) { + return false } const newHdrs = await ws.listAsync() - const newLastModTime = computeLastModTime(newHdrs); - if (newLastModTime === oldModTime) { + const newHdrsHash = computeHeadersHash(newHdrs); + if (newHdrsHash === oldHdrsHash) { // no change, keep the old cache cacheHdrs = oldHdrs cacheHdrsMap = oldHdrsMap @@ -111,7 +119,7 @@ export function createCachedWorkspace(ws: WorkspaceProvider): CachedWorkspacePro } // save results - cacheModTime = newLastModTime + cacheHdrsHash = newHdrsHash cacheProjs = newProjs cacheHdrs = newHdrs cacheHdrsMap = newHdrsMap @@ -141,14 +149,13 @@ export function createCachedWorkspace(ws: WorkspaceProvider): CachedWorkspacePro text, version: prevVer } - // update headers list + // update headers list, map and hash if (!cacheHdrsMap[h.id]) { cacheHdrs.push(h) } - // update headers map cacheHdrsMap[h.id] = h - // update mod time - cacheModTime = Math.max(cacheModTime, h.modificationTime) + cacheHdrsHash = computeHeadersHash(cacheHdrs) + // send update to backing storage const res = await ws.setAsync(h, prevVer, text) if (res) { // update cached projec @@ -160,17 +167,21 @@ export function createCachedWorkspace(ws: WorkspaceProvider): CachedWorkspacePro } else { // conflict; delete cache delete cacheProjs[h.id] + // TODO @darzu: fix header(s) after conflict ? } return res; } async function deleteAsync(h: Header, prevVer: Version): Promise { await pendingUpdate; + // update cached projects delete cacheProjs[h.id]; - // TODO @darzu: how to handle mod time with delete? - // TODO @darzu: we should probably enforce soft delete everywhere... - cacheModTime = Math.max(cacheModTime, h.modificationTime) - const res = await ws.deleteAsync(h, prevVer) - return res; + // update headers list, map and hash + delete cacheHdrsMap[h.id]; + cacheHdrs = cacheHdrs.filter(r => r.id === h.id); + cacheHdrsHash = computeHeadersHash(cacheHdrs); + // send update to backing storage + await ws.deleteAsync(h, prevVer) + // TODO @darzu: fix header(s) after conflict ? } async function resetAsync() { await pendingUpdate; @@ -180,12 +191,12 @@ export function createCachedWorkspace(ws: WorkspaceProvider): CachedWorkspacePro const provider: CachedWorkspaceProvider = { // cache - getLastModTime, + getHeadersHash, synchronize, pendingSync: () => pendingUpdate, firstSync: () => firstUpdate, listSync: () => cacheHdrs, - hasSync: h => !!cacheHdrsMap[h.id], + hasSync: h => h && !!cacheHdrsMap[h.id], tryGetSync: h => cacheProjs[h.id], // workspace getAsync, @@ -207,21 +218,21 @@ export function createCloudSyncWorkspace(cloud: WorkspaceProvider, cloudLocal: W const firstCachePull = Promise.all([cloudCache.firstSync(), localCache.firstSync()]) const pendingCacheSync = () => Promise.all([cloudCache.pendingSync(), localCache.pendingSync()]) - const getLastModTime = () => Math.max(cloudCache.getLastModTime(), localCache.getLastModTime()) - const needsSync = () => cloudCache.getLastModTime() !== localCache.getLastModTime() + const getHeadersHash = () => localCache.getHeadersHash() + const needsSync = () => cloudCache.getHeadersHash() !== localCache.getHeadersHash() - // TODO @darzu: we could frequently check the last mod times to see if a sync is in order? + // TODO @darzu: we could frequently check the last mod times to see if a sync is in order? // TODO @darzu: multi-tab safety for cloudLocal // TODO @darzu: when two workspaces disagree on last mod time, we should sync? - let pendingSync: Promise = synchronizeInternal(); - const firstSync = pendingSync; + const firstSync = synchronizeInternal({pollStorage: true});; + let pendingSync = firstSync; - async function synchronize(expectedLastModTime?:number): Promise { + async function synchronize(reason: SynchronizationReason): Promise { if (pendingSync.isPending()) return pendingSync - pendingSync = synchronizeInternal(expectedLastModTime) + pendingSync = synchronizeInternal(reason) return pendingSync } @@ -260,7 +271,7 @@ export function createCloudSyncWorkspace(cloud: WorkspaceProvider, cloudLocal: W return newH; } - async function synchronizeInternal(expectedLastModTime?:number): Promise { + async function synchronizeInternal(reason: SynchronizationReason): Promise { console.log("cloudsyncworkspace: synchronizeInternal") // TODO @darzu: error: circular promise resolution chain @@ -290,12 +301,7 @@ export function createCloudSyncWorkspace(cloud: WorkspaceProvider, cloudLocal: W // } // wait for each side to sync - // TODO @darzu: this isn't symmetric. Can we fix the abstraction so it is? - if (!expectedLastModTime) { - await Promise.all([cloudCache.synchronize(), localCache.pendingSync()]) - } else { - await Promise.all([cloudCache.pendingSync(), localCache.synchronize(expectedLastModTime)]) - } + await Promise.all([cloudCache.synchronize(reason), localCache.synchronize(reason)]) // TODO @darzu: mod time isn't sufficient for a set of headers; maybe hash or merkle tree // // short circuit if there aren't changes @@ -389,7 +395,7 @@ export function createCloudSyncWorkspace(cloud: WorkspaceProvider, cloudLocal: W const provider: CloudSyncWorkspace = { // cache - getLastModTime, + getHeadersHash, synchronize, pendingSync: () => pendingSync, firstSync: () => firstSync, diff --git a/webapp/src/workspaces/jointworkspace.ts b/webapp/src/workspaces/jointworkspace.ts index 0ad37e45114..8492208ac86 100644 --- a/webapp/src/workspaces/jointworkspace.ts +++ b/webapp/src/workspaces/jointworkspace.ts @@ -1,4 +1,4 @@ -import { CachedWorkspaceProvider } from "./cloudsyncworkspace"; +import { CachedWorkspaceProvider, SynchronizationReason } from "./cloudsyncworkspace"; type Header = pxt.workspace.Header; type ScriptText = pxt.workspace.ScriptText; @@ -68,11 +68,12 @@ export function createJointWorkspace(...all: CachedWorkspaceProvider[]): CachedW const firstSync = async () => (await Promise.all(all.map(w => w.firstSync()))).reduce((p, n) => p || n, false) const pendingSync = async () => (await Promise.all(all.map(w => w.pendingSync()))).reduce((p, n) => p || n, false) - const getLastModTime = () => Math.max(...all.map(w => w.getLastModTime())) + // TODO @darzu: is this too expensive? + const getHeadersHash = () => all.map(w => w.getHeadersHash()).join("|") - async function synchronize(expectedLastModTime?: number): Promise { - return (await Promise.all(all.map(w => w.synchronize(expectedLastModTime)))) - .reduce((p, n) => p || n, false) + async function synchronize(reason: SynchronizationReason): Promise { + const changes = await Promise.all(all.map(w => w.synchronize(reason))) + return changes.reduce((p, n) => p || n, false) } function listSync(): Header[] { // return all (assuming disjoint) @@ -104,6 +105,7 @@ export function createJointWorkspace(...all: CachedWorkspaceProvider[]): CachedW } async function setAsync(h: Header, prevVer: any, text?: ScriptText): Promise { await pendingSync() + // TODO @darzu: dbg logging console.log("joint:setAsync") console.dir(all.map(w => w.hasSync(h))) const ws = getWorkspaceFor(h) ?? all[0] @@ -121,7 +123,7 @@ export function createJointWorkspace(...all: CachedWorkspaceProvider[]): CachedW const provider: CachedWorkspaceProvider = { // cache - getLastModTime, + getHeadersHash, synchronize, pendingSync, firstSync, diff --git a/webapp/src/workspaces/workspace.ts b/webapp/src/workspaces/workspace.ts index eba177a2fe6..7ae7a6ef586 100644 --- a/webapp/src/workspaces/workspace.ts +++ b/webapp/src/workspaces/workspace.ts @@ -49,6 +49,7 @@ type File = pxt.workspace.File; // data.invalidate("gh-commits:*"); // invalidate commits just in case // [ ] remove syncAsync // [ ] ensure we don't regress https://github.com/microsoft/pxt/issues/7520 +// [ ] add analytics & hueristics for detecting project loss // TODO @darzu: remove. redudant w/ implCache @@ -171,7 +172,7 @@ export function setupWorkspace(kind: WorkspaceKind): void { console.log("synchronizing with the cloud..."); console.log("before:") console.dir(joint.listSync().map(j => ({id: j.id, t: j.modificationTime}))) - const changed = await joint.synchronize() + const changed = await joint.synchronize({pollStorage: true}) if (changed) { console.log("after:") console.dir(joint.listSync().map(j => ({id: j.id, t: j.modificationTime}))) @@ -335,30 +336,23 @@ export function getHeader(id: string) { // this key is the max modificationTime value of the allHeaders // it is used to track if allHeaders need to be refreshed (syncAsync) -let sessionID: string = ""; +let _allHeadersSessionHash: string = ""; export function isHeadersSessionOutdated() { - return pxt.storage.getLocal('workspacesessionid') != sessionID; + return pxt.storage.getLocal('workspacesessionid') != _allHeadersSessionHash; } function maybeSyncHeadersAsync(): Promise { if (isHeadersSessionOutdated()) // another tab took control return syncAsync().then(() => { }) // take back control return Promise.resolve(); } -function computeHeadersSessionID() { - const hdrs = impl.listSync() - return hdrs.length + ' ' + hdrs - .map(h => h.modificationTime) - .reduce((l, r) => Math.max(l, r), 0) - .toString() -} function refreshHeadersSession() { // TODO @darzu: carefully handle this. // use # of scripts + time of last mod as key - sessionID = computeHeadersSessionID(); + _allHeadersSessionHash = impl.getHeadersHash(); if (isHeadersSessionOutdated()) { - pxt.storage.setLocal('workspacesessionid', sessionID); - pxt.debug(`workspace: refreshed headers session to ${sessionID}`); + pxt.storage.setLocal('workspacesessionid', _allHeadersSessionHash); + pxt.debug(`workspace: refreshed headers session to ${_allHeadersSessionHash}`); data.invalidate("header:*"); data.invalidate("text:*"); } @@ -1561,16 +1555,23 @@ export async function saveToCloudAsync(h: Header) { // return cloudsync.saveToCloudAsync(h) } - export async function syncAsync(): Promise { + // contract: + // output: this tab's headers session is up to date ... + // TODO @darzu: ... and re-acquires headers ? // TODO @darzu: clean up naming, layering // TODO @darzu: handle: // filters?: pxt.editor.ProjectFilters; // searchBar?: boolean; + const storedSessionID = pxt.storage.getLocal('workspacesessionid') + const memSessionID = _allHeadersSessionHash; + const syncReason = { + localStorageDesync: true, + cloudPoll: false, + } return syncAsync2() } - // this promise is set while a sync is in progress // cleared when sync is done. let syncAsyncPromise: Promise; From 49fb645954f98191b0a067fdc51ce6ca4f294132 Mon Sep 17 00:00:00 2001 From: darzu Date: Tue, 29 Dec 2020 11:11:30 -0800 Subject: [PATCH 43/52] todo list and progress on syncAsync --- webapp/src/workspaces/cloudsyncworkspace.ts | 16 +---- webapp/src/workspaces/jointworkspace.ts | 17 ++++- webapp/src/workspaces/workspace.ts | 70 ++++++++++++++++++--- 3 files changed, 77 insertions(+), 26 deletions(-) diff --git a/webapp/src/workspaces/cloudsyncworkspace.ts b/webapp/src/workspaces/cloudsyncworkspace.ts index 62808069d94..95a3a4df275 100644 --- a/webapp/src/workspaces/cloudsyncworkspace.ts +++ b/webapp/src/workspaces/cloudsyncworkspace.ts @@ -274,8 +274,7 @@ export function createCloudSyncWorkspace(cloud: WorkspaceProvider, cloudLocal: W async function synchronizeInternal(reason: SynchronizationReason): Promise { console.log("cloudsyncworkspace: synchronizeInternal") - // TODO @darzu: error: circular promise resolution chain - + // TODO @darzu: review these cases: // case 1: everything should be synced up, we're just polling the server // expectedLastModTime = 0 // we definitely want cloudCache to synchronize @@ -293,22 +292,9 @@ export function createCloudSyncWorkspace(cloud: WorkspaceProvider, cloudLocal: W // we want to wait on localCache.pendingSync // TODO @darzu: need to think through and compare how this would work with git - // TODO @darzu: not sure what case would hit this: - // if (!expectedLastModTime) { - // // first check if there is a known disagreement before we force each side to deep sync - // if (cloudCache.getLastModTime() !== localCache.getLastModTime()) - // expectedLastModTime = getLastModTime(); - // } - // wait for each side to sync await Promise.all([cloudCache.synchronize(reason), localCache.synchronize(reason)]) - // TODO @darzu: mod time isn't sufficient for a set of headers; maybe hash or merkle tree - // // short circuit if there aren't changes - // if (cloudCache.getLastModTime() === localCache.getLastModTime()) { - // return false - // } - // TODO @darzu: re-generalize? const left = cloudCache; const right = localCache; diff --git a/webapp/src/workspaces/jointworkspace.ts b/webapp/src/workspaces/jointworkspace.ts index 8492208ac86..5d49508a879 100644 --- a/webapp/src/workspaces/jointworkspace.ts +++ b/webapp/src/workspaces/jointworkspace.ts @@ -60,6 +60,15 @@ export function createJointWorkspace2(primary: WorkspaceProvider, ...others: Wor return provider; } +// note: these won't work recursively, but as of now there's no forseen use +// case beyond 1 level. If needed, we could use a hash tree/merkle tree. +function joinHdrsHash(...hashes: string[]): string { + return hashes?.join("|") ?? "" +} +function splitHdrsHash(hash: string): string[] { + return hash?.split("|") ?? [] +} + export function createJointWorkspace(...all: CachedWorkspaceProvider[]): CachedWorkspaceProvider { // TODO @darzu: we're assuming they are disjoint for now @@ -69,10 +78,14 @@ export function createJointWorkspace(...all: CachedWorkspaceProvider[]): CachedW const firstSync = async () => (await Promise.all(all.map(w => w.firstSync()))).reduce((p, n) => p || n, false) const pendingSync = async () => (await Promise.all(all.map(w => w.pendingSync()))).reduce((p, n) => p || n, false) // TODO @darzu: is this too expensive? - const getHeadersHash = () => all.map(w => w.getHeadersHash()).join("|") + const getHeadersHash = () => joinHdrsHash(...all.map(w => w.getHeadersHash())) async function synchronize(reason: SynchronizationReason): Promise { - const changes = await Promise.all(all.map(w => w.synchronize(reason))) + const expectedHashes = splitHdrsHash(reason.expectedHeadersHash) + const changes = await Promise.all(all.map((w, i) => w.synchronize({ + ...reason, + expectedHeadersHash: expectedHashes[i] + }))) return changes.reduce((p, n) => p || n, false) } function listSync(): Header[] { diff --git a/webapp/src/workspaces/workspace.ts b/webapp/src/workspaces/workspace.ts index 7ae7a6ef586..663a73c3a45 100644 --- a/webapp/src/workspaces/workspace.ts +++ b/webapp/src/workspaces/workspace.ts @@ -38,7 +38,7 @@ type InstallHeader = pxt.workspace.InstallHeader; type File = pxt.workspace.File; // TODO @darzu: todo list: -// [ ] remove / fix header session methods +// [x] remove / fix header session methods // [ ] remove forceSaveAsync // [x] remove allScripts // [ ] remove headerQ @@ -47,9 +47,48 @@ type File = pxt.workspace.File; // data.invalidateHeader("text", hd); // data.invalidateHeader("pkg-git-status", hd); // data.invalidate("gh-commits:*"); // invalidate commits just in case -// [ ] remove syncAsync +// [x] remove syncAsync // [ ] ensure we don't regress https://github.com/microsoft/pxt/issues/7520 // [ ] add analytics & hueristics for detecting project loss +// [ ] handle switchToMemoryWorkspace +// [ ] soft delete, ensure we are prefering +// [ ] don't block on network +// [ ] 1st load +// [ ] ever +// [ ] 1st time migrate local -> online +// [ ] multi-user seperation +// [ ] client can't change/delete other user content +// [ ] background work: +// [ ] queueing +// [ ] updating in the queue +// [ ] thorttle, debounce +// [ ] batch +// [ ] don't sync when tab idle, or sync exp backoff +// [ ] cloud state UX +// [ ] project list +// [ ] conflict resolution dialog +// [ ] in editor +// [ ] clean up code +// [ ] handle all TODO @darzu's +// [ ] renames +// [ ] synchronize -> syncAsync +// [ ] cloudsyncworkspace, +// [ ] cloudsync, +// [ ] oldbrowserdbworkspace, +// [ ] synchronizedworkspace, +// [ ] workspacebehavior +// TESTING: +// for each: +// [ ] create new prj +// [ ] delete prj +// [ ] mod prj +// [ ] reset +// do: +// [ ] online +// [ ] offline, signed in +// [ ] offline, signed out +// [ ] multi-tab +// [ ] multi-browser // TODO @darzu: remove. redudant w/ implCache @@ -1556,20 +1595,33 @@ export async function saveToCloudAsync(h: Header) { } export async function syncAsync(): Promise { + console.log("workspace:syncAsync"); // contract: // output: this tab's headers session is up to date ... // TODO @darzu: ... and re-acquires headers ? // TODO @darzu: clean up naming, layering + const expectedHeadersHash = pxt.storage.getLocal('workspacesessionid') + const changed = await impl.synchronize({ + expectedHeadersHash, + }) // TODO @darzu: handle: // filters?: pxt.editor.ProjectFilters; // searchBar?: boolean; - const storedSessionID = pxt.storage.getLocal('workspacesessionid') - const memSessionID = _allHeadersSessionHash; - const syncReason = { - localStorageDesync: true, - cloudPoll: false, - } - return syncAsync2() + + // TODO @darzu: \/ + /* + // force reload + ex.text = undefined + ex.version = undefined + + data.invalidateHeader("header", hd); + data.invalidateHeader("text", hd); + data.invalidateHeader("pkg-git-status", hd); + data.invalidate("gh-commits:*"); // invalidate commits just in case + + impl.getSyncState() + */ + return {} } // this promise is set while a sync is in progress From 6b388a828e4711e124cecbeea83fed93a14e4a44 Mon Sep 17 00:00:00 2001 From: darzu Date: Sat, 2 Jan 2021 09:50:00 -0800 Subject: [PATCH 44/52] return headers[] from sync; finer grain virtual api inval --- webapp/src/app.tsx | 2 +- webapp/src/cloud.ts | 3 +- webapp/src/workspaces/browserdbworkspace.ts | 3 +- webapp/src/workspaces/cloudsyncworkspace.ts | 53 ++++++++-------- webapp/src/workspaces/jointworkspace.ts | 11 ++-- .../src/workspaces/synchronizedworkspace.ts | 51 ---------------- webapp/src/workspaces/workspace.ts | 60 ++++++++++++------- webapp/src/workspaces/workspacebehavior.ts | 14 ++--- 8 files changed, 85 insertions(+), 112 deletions(-) delete mode 100644 webapp/src/workspaces/synchronizedworkspace.ts diff --git a/webapp/src/app.tsx b/webapp/src/app.tsx index f07af9484bf..0c986a576fc 100644 --- a/webapp/src/app.tsx +++ b/webapp/src/app.tsx @@ -263,7 +263,7 @@ export class ProjectView } this.saveFileAsync().done(); } else if (active) { - // TODO @darzu: new code path: + // TODO @darzu: new code path maybe: // workspace.syncAsync().then(changed => this.reloadAsync()) // reloadAsync: this.loadHeaderAsync() // OR: subscribe to data api, on change, reload diff --git a/webapp/src/cloud.ts b/webapp/src/cloud.ts index d466f037842..ca5db194f5b 100644 --- a/webapp/src/cloud.ts +++ b/webapp/src/cloud.ts @@ -10,6 +10,7 @@ type Header = pxt.workspace.Header; type ScriptText = pxt.workspace.ScriptText; import U = pxt.Util; +import { toDbg } from "./workspaces/cloudsyncworkspace"; const state = { uploadCount: 0, @@ -38,7 +39,7 @@ export async function listAsync(): Promise { return header; }); // TODO @darzu: dbg - console.dir(headers.map(h => ({h: h.id, t: h.modificationTime}))) + console.dir(headers.map(toDbg)) resolve(headers); } else { reject(new Error(result.errmsg)); diff --git a/webapp/src/workspaces/browserdbworkspace.ts b/webapp/src/workspaces/browserdbworkspace.ts index 93365a2097c..8e0e66c1ab5 100644 --- a/webapp/src/workspaces/browserdbworkspace.ts +++ b/webapp/src/workspaces/browserdbworkspace.ts @@ -1,4 +1,5 @@ import * as db from "../db"; +import { toDbg } from "./cloudsyncworkspace"; type Header = pxt.workspace.Header; type ScriptText = pxt.workspace.ScriptText; @@ -33,7 +34,7 @@ export function createBrowserDbWorkspace(namespace: string): BrowserDbWorkspaceP const hdrs: pxt.workspace.Header[] = await headerDb.getAllAsync(); // const txts: TextDbEntry[] = await textDb.getAllAsync(); console.log(`dbg ${prefix}-headers:`); - console.dir(hdrs.map(h => ({id: h.id, t: h.modificationTime}))) + console.dir(hdrs.map(toDbg)) } // TODO @darzu: dbg printDbg(); diff --git a/webapp/src/workspaces/cloudsyncworkspace.ts b/webapp/src/workspaces/cloudsyncworkspace.ts index 95a3a4df275..979d44d3f6f 100644 --- a/webapp/src/workspaces/cloudsyncworkspace.ts +++ b/webapp/src/workspaces/cloudsyncworkspace.ts @@ -15,9 +15,9 @@ type WorkspaceProvider = pxt.workspace.WorkspaceProvider; export interface CachedWorkspaceProvider extends WorkspaceProvider { getHeadersHash(): string, - synchronize(reason: SynchronizationReason): Promise, // TODO @darzu: name syncAsync? - pendingSync(): Promise, - firstSync(): Promise, + synchronize(reason: SynchronizationReason): Promise, // TODO @darzu: name syncAsync? + pendingSync(): Promise, + firstSync(): Promise, listSync(): Header[], hasSync(h: Header): boolean, tryGetSync(h: Header): WsFile @@ -39,8 +39,8 @@ function computeHeadersHash(hdrs: Header[]): string { } function hasChanged(a: Header, b: Header): boolean { - // TODO @darzu: use e-tag, _rev, or version uuid instead? - return a.modificationTime !== b.modificationTime + // TODO @darzu: use e-tag, _rev, version uuid, or hash instead? + return (!!a !== !!b) || a?.modificationTime !== b?.modificationTime } // TODO @darzu: use cases: multi-tab and cloud @@ -63,9 +63,9 @@ export function createCachedWorkspace(ws: WorkspaceProvider): CachedWorkspacePro } // TODO @darzu: do we want to kick off the first sync at construction? Side-effects at construction are usually bad.. - let pendingUpdate: Promise = synchronizeInternal({ pollStorage: true }); - const firstUpdate = pendingUpdate; - async function synchronize(reason: SynchronizationReason): Promise { + const firstUpdate = synchronizeInternal({ pollStorage: true }); + let pendingUpdate = firstUpdate; + async function synchronize(reason: SynchronizationReason): Promise { if (pendingUpdate.isPending()) return pendingUpdate pendingUpdate = synchronizeInternal(reason) @@ -80,7 +80,7 @@ export function createCachedWorkspace(ws: WorkspaceProvider): CachedWorkspacePro cacheHdrsHash = "" } - async function synchronizeInternal(reason: SynchronizationReason): Promise { + async function synchronizeInternal(reason: SynchronizationReason): Promise { // remember our old cache, we might keep items from it later const oldHdrs = cacheHdrs const oldHdrsMap = cacheHdrsMap @@ -93,7 +93,7 @@ export function createCachedWorkspace(ws: WorkspaceProvider): CachedWorkspacePro // TODO @darzu: does this buy us anything? eraseCache() } else if (!needSync) { - return false + return [] } const newHdrs = await ws.listAsync() @@ -103,19 +103,18 @@ export function createCachedWorkspace(ws: WorkspaceProvider): CachedWorkspacePro cacheHdrs = oldHdrs cacheHdrsMap = oldHdrsMap cacheProjs = oldProjs - return false + return [] } console.log("cachedworkspace: synchronizeInternal (1)") // TODO @darzu: dbg // compute header differences and clear old cache entries const newHdrsMap = U.toDictionary(newHdrs, h => h.id) - const newProjs = oldProjs - for (let id of Object.keys(newProjs)) { - const newHdr = newHdrsMap[id] - if (!newHdr || hasChanged(newHdr, oldHdrsMap[id])) { - console.log(`DELETE ${id}`) // TODO @darzu: dbg - delete newProjs[id] - } + const changedHdrIds = U.unique([...oldHdrs, ...newHdrs], h => h.id).map(h => h.id) + .filter(id => hasChanged(oldHdrsMap[id], newHdrsMap[id])) + const newProjs = oldProjs // TODO @darzu: is there any point in copying here? + for (let id of changedHdrIds) { + console.log(`cache invalidating ${id}`) // TODO @darzu: dbg + delete newProjs[id] } // save results @@ -123,7 +122,7 @@ export function createCachedWorkspace(ws: WorkspaceProvider): CachedWorkspacePro cacheProjs = newProjs cacheHdrs = newHdrs cacheHdrsMap = newHdrsMap - return true; + return changedHdrIds.map(i => newHdrsMap[i]); } async function listAsync(): Promise { @@ -209,6 +208,11 @@ export function createCachedWorkspace(ws: WorkspaceProvider): CachedWorkspacePro return provider; } +// TODO @darzu: dbging helper +export function toDbg(h: Header) { + return {n: h.name, t: h.modificationTime, id: h.id} +} + export interface CloudSyncWorkspace extends CachedWorkspaceProvider { } @@ -229,7 +233,7 @@ export function createCloudSyncWorkspace(cloud: WorkspaceProvider, cloudLocal: W const firstSync = synchronizeInternal({pollStorage: true});; let pendingSync = firstSync; - async function synchronize(reason: SynchronizationReason): Promise { + async function synchronize(reason: SynchronizationReason): Promise { if (pendingSync.isPending()) return pendingSync pendingSync = synchronizeInternal(reason) @@ -271,7 +275,7 @@ export function createCloudSyncWorkspace(cloud: WorkspaceProvider, cloudLocal: W return newH; } - async function synchronizeInternal(reason: SynchronizationReason): Promise { + async function synchronizeInternal(reason: SynchronizationReason): Promise { console.log("cloudsyncworkspace: synchronizeInternal") // TODO @darzu: review these cases: @@ -337,10 +341,11 @@ export function createCloudSyncWorkspace(cloud: WorkspaceProvider, cloudLocal: W // wait // TODO @darzu: worklist? batching? throttling? incremental? - const changed = await Promise.all([...lPushPromises, ...rPushPromises]) + const allPushes = await Promise.all([...lPushPromises, ...rPushPromises]) + const changes = U.unique(allPushes, h => h.id) // TODO @darzu: what about mod time changes? - return changed.length >= 0; + return changes } async function listAsync(): Promise { @@ -376,7 +381,7 @@ export function createCloudSyncWorkspace(cloud: WorkspaceProvider, cloudLocal: W // TODO @darzu: debug logging firstSync.then(c => { console.log("cloudSyncWS first update:") - console.dir(localCache.listSync().map(h => ({id: h.id, t: h.modificationTime}))) + console.dir(localCache.listSync().map(toDbg)) }) const provider: CloudSyncWorkspace = { diff --git a/webapp/src/workspaces/jointworkspace.ts b/webapp/src/workspaces/jointworkspace.ts index 5d49508a879..40b57bcc0ca 100644 --- a/webapp/src/workspaces/jointworkspace.ts +++ b/webapp/src/workspaces/jointworkspace.ts @@ -4,6 +4,7 @@ type Header = pxt.workspace.Header; type ScriptText = pxt.workspace.ScriptText; type File = pxt.workspace.File; type WorkspaceProvider = pxt.workspace.WorkspaceProvider; +import U = pxt.Util; async function unique(...listFns: (() => Promise)[]) { const allHdrs = (await Promise.all(listFns.map(ls => ls()))) @@ -75,18 +76,20 @@ export function createJointWorkspace(...all: CachedWorkspaceProvider[]): CachedW // TODO @darzu: debug logging console.log(`createJointWorkspace`); - const firstSync = async () => (await Promise.all(all.map(w => w.firstSync()))).reduce((p, n) => p || n, false) - const pendingSync = async () => (await Promise.all(all.map(w => w.pendingSync()))).reduce((p, n) => p || n, false) + const flattenAndUniqueHdrs = (hs: Header[][]) => U.unique(hs.reduce((p, n) => [...p, ...n], []), h => h.id) + + const firstSync = async () => flattenAndUniqueHdrs(await Promise.all(all.map(w => w.firstSync()))) + const pendingSync = async () => flattenAndUniqueHdrs(await Promise.all(all.map(w => w.pendingSync()))) // TODO @darzu: is this too expensive? const getHeadersHash = () => joinHdrsHash(...all.map(w => w.getHeadersHash())) - async function synchronize(reason: SynchronizationReason): Promise { + async function synchronize(reason: SynchronizationReason): Promise { const expectedHashes = splitHdrsHash(reason.expectedHeadersHash) const changes = await Promise.all(all.map((w, i) => w.synchronize({ ...reason, expectedHeadersHash: expectedHashes[i] }))) - return changes.reduce((p, n) => p || n, false) + return flattenAndUniqueHdrs(changes) } function listSync(): Header[] { // return all (assuming disjoint) diff --git a/webapp/src/workspaces/synchronizedworkspace.ts b/webapp/src/workspaces/synchronizedworkspace.ts deleted file mode 100644 index 96152da63fb..00000000000 --- a/webapp/src/workspaces/synchronizedworkspace.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { SyncWorkspaceProvider } from "./memworkspace"; -import { ConflictStrategy, DisjointSetsStrategy, Strategy, synchronize } from "./workspacebehavior"; - -type Header = pxt.workspace.Header; -type ScriptText = pxt.workspace.ScriptText; -type File = pxt.workspace.File; -type WorkspaceProvider = pxt.workspace.WorkspaceProvider; - - -export interface Synchronizable { - syncAsync(): Promise, - // TODO @darzu: - // lastLeftList(): Header[], - // lastRightList(): Header[], -} - -export function createSynchronizedWorkspace(primary: WorkspaceProvider, cache: T, strat: Strategy): T & Synchronizable { - - // TODO @darzu: debated caching items here - // const lastLeft: Header[] = []; - // const lastRight: Header[] = []; - - async function syncAsync() { - // TODO @darzu: parameterize strategy? - const res = await synchronize(primary, cache, strat) - return res.changed - } - - return { - ...cache, - // TODO @darzu: - // listAsync: async () => { - - // }, - // mutative operations should be kicked off for both - setAsync: async (h, prevVersion, text) => { - // TODO @darzu: don't push to both when disjoint sets strat isn't synchronize - const a = primary.setAsync(h, prevVersion, text) - const b = cache.setAsync(h, prevVersion, text) - await Promise.all([a,b]) - return await a; - }, - deleteAsync: async (h, prevVersion) => { - const a = primary.deleteAsync(h, prevVersion) - const b = cache.deleteAsync(h, prevVersion) - await Promise.all([a,b]) - return await a; - }, - syncAsync, - }; -} diff --git a/webapp/src/workspaces/workspace.ts b/webapp/src/workspaces/workspace.ts index 663a73c3a45..e1816149846 100644 --- a/webapp/src/workspaces/workspace.ts +++ b/webapp/src/workspaces/workspace.ts @@ -22,10 +22,8 @@ import U = pxt.Util; import Cloud = pxt.Cloud; import { createJointWorkspace, createJointWorkspace2 } from "./jointworkspace"; import { createBrowserDbWorkspace } from "./browserdbworkspace"; -import { createSynchronizedWorkspace, Synchronizable } from "./synchronizedworkspace"; -import { ConflictStrategy, DisjointSetsStrategy, migrateOverlap, wrapInMemCache } from "./workspacebehavior"; import { createMemWorkspace, SyncWorkspaceProvider } from "./memworkspace"; -import { CachedWorkspaceProvider, createCachedWorkspace, createCloudSyncWorkspace } from "./cloudsyncworkspace"; +import { CachedWorkspaceProvider, createCachedWorkspace, createCloudSyncWorkspace, toDbg } from "./cloudsyncworkspace"; // Avoid importing entire crypto-js /* tslint:disable:no-submodule-imports */ @@ -38,18 +36,21 @@ type InstallHeader = pxt.workspace.InstallHeader; type File = pxt.workspace.File; // TODO @darzu: todo list: -// [x] remove / fix header session methods -// [ ] remove forceSaveAsync -// [x] remove allScripts -// [ ] remove headerQ +// [ ] remove / fix header session methods +// [ ] refreshHeadersSession // [ ] invalidate virtual api: // data.invalidateHeader("header", hd); // data.invalidateHeader("text", hd); // data.invalidateHeader("pkg-git-status", hd); // data.invalidate("gh-commits:*"); // invalidate commits just in case +// [x] understand commitAsync +// [ ] remove forceSaveAsync +// [x] remove allScripts +// [ ] remove headerQ // [x] remove syncAsync // [ ] ensure we don't regress https://github.com/microsoft/pxt/issues/7520 -// [ ] add analytics & hueristics for detecting project loss +// [ ] add analytics +// [ ] hueristics for detecting project loss // [ ] handle switchToMemoryWorkspace // [ ] soft delete, ensure we are prefering // [ ] don't block on network @@ -68,6 +69,7 @@ type File = pxt.workspace.File; // [ ] project list // [ ] conflict resolution dialog // [ ] in editor +// [ ] refactor out git / github stuff // [ ] clean up code // [ ] handle all TODO @darzu's // [ ] renames @@ -159,7 +161,6 @@ function chooseWorkspace(kind: WorkspaceKind = "browser"): pxt.workspace.Workspa } } - export function setupWorkspace(kind: WorkspaceKind): void { U.assert(!impl, "workspace set twice"); pxt.log(`workspace: ${kind}`); @@ -200,21 +201,18 @@ export function setupWorkspace(kind: WorkspaceKind): void { // TODO @darzu: improve this const msPerMin = 1000 * 60 - const afterSync = (changed: boolean) => { - console.log(`...changes synced! ${changed}`) - if (changed) { - data.invalidate("header:*"); - data.invalidate("text:*"); - } + const afterSync = (changed: Header[]) => { + console.log(`...changes synced! ${!!changed}`) + onExternalHeaderChanges(changed) } const doSync = async () => { console.log("synchronizing with the cloud..."); console.log("before:") - console.dir(joint.listSync().map(j => ({id: j.id, t: j.modificationTime}))) + console.dir(joint.listSync().map(toDbg)) const changed = await joint.synchronize({pollStorage: true}) if (changed) { console.log("after:") - console.dir(joint.listSync().map(j => ({id: j.id, t: j.modificationTime}))) + console.dir(joint.listSync().map(toDbg)) } afterSync(changed) } @@ -376,16 +374,18 @@ export function getHeader(id: string) { // this key is the max modificationTime value of the allHeaders // it is used to track if allHeaders need to be refreshed (syncAsync) let _allHeadersSessionHash: string = ""; +// TODO @darzu: delete this (unneeded) export function isHeadersSessionOutdated() { return pxt.storage.getLocal('workspacesessionid') != _allHeadersSessionHash; } +// TODO @darzu: delete this (unneeded) function maybeSyncHeadersAsync(): Promise { if (isHeadersSessionOutdated()) // another tab took control return syncAsync().then(() => { }) // take back control return Promise.resolve(); } +// TODO @darzu: delete this (unused) function refreshHeadersSession() { - // TODO @darzu: carefully handle this. // use # of scripts + time of last mod as key _allHeadersSessionHash = impl.getHeadersHash(); @@ -1594,6 +1594,22 @@ export async function saveToCloudAsync(h: Header) { // return cloudsync.saveToCloudAsync(h) } +// called when external changes happen to our headers (e.g. multi-tab +// scenarios, cloud sync, etc.) +function onExternalHeaderChanges(changedHdrs: Header[]) { + changedHdrs.forEach(hd => { + data.invalidateHeader("header", hd); + data.invalidateHeader("text", hd); + data.invalidateHeader("pkg-git-status", hd); + }) + if (changedHdrs.length) { + // TODO @darzu: can we make this more fine grain? + data.invalidate("gh-commits:*"); // invalidate commits just in case + console.log(`onExternalHeaderChanges:`) + console.dir(changedHdrs.map(toDbg)) // TODO @darzu: dbg + } +} + export async function syncAsync(): Promise { console.log("workspace:syncAsync"); // contract: @@ -1601,9 +1617,11 @@ export async function syncAsync(): Promise { // TODO @darzu: ... and re-acquires headers ? // TODO @darzu: clean up naming, layering const expectedHeadersHash = pxt.storage.getLocal('workspacesessionid') - const changed = await impl.synchronize({ + const changedHdrs = await impl.synchronize({ expectedHeadersHash, }) + onExternalHeaderChanges(changedHdrs); + pxt.storage.setLocal('workspacesessionid', impl.getHeadersHash()); // TODO @darzu: handle: // filters?: pxt.editor.ProjectFilters; // searchBar?: boolean; @@ -1614,10 +1632,6 @@ export async function syncAsync(): Promise { ex.text = undefined ex.version = undefined - data.invalidateHeader("header", hd); - data.invalidateHeader("text", hd); - data.invalidateHeader("pkg-git-status", hd); - data.invalidate("gh-commits:*"); // invalidate commits just in case impl.getSyncState() */ diff --git a/webapp/src/workspaces/workspacebehavior.ts b/webapp/src/workspaces/workspacebehavior.ts index 20d897d511e..4789812e9bf 100644 --- a/webapp/src/workspaces/workspacebehavior.ts +++ b/webapp/src/workspaces/workspacebehavior.ts @@ -1,5 +1,4 @@ import { createMemWorkspace, SyncWorkspaceProvider } from "./memworkspace"; -import { createSynchronizedWorkspace, Synchronizable } from "./synchronizedworkspace"; import U = pxt.Util; type WorkspaceProvider = pxt.workspace.WorkspaceProvider; @@ -135,12 +134,13 @@ export async function synchronize(left: WorkspaceProvider, right: WorkspaceProvi } } -export function wrapInMemCache(ws: WorkspaceProvider): SyncWorkspaceProvider & WorkspaceProvider & Synchronizable { - return createSynchronizedWorkspace(ws, createMemWorkspace(), { - conflict: ConflictStrategy.LastWriteWins, - disjointSets: DisjointSetsStrategy.Synchronize - }); -} +// TODO @darzu: use or delete +// export function wrapInMemCache(ws: WorkspaceProvider): SyncWorkspaceProvider & WorkspaceProvider & Synchronizable { +// return createSynchronizedWorkspace(ws, createMemWorkspace(), { +// conflict: ConflictStrategy.LastWriteWins, +// disjointSets: DisjointSetsStrategy.Synchronize +// }); +// } export async function migrateOverlap(fromWs: WorkspaceProvider, toWs: WorkspaceProvider) { // TODO @darzu: From e970d5303761f963ce8032c39355c27a0b4cd01c Mon Sep 17 00:00:00 2001 From: darzu Date: Sat, 2 Jan 2021 10:04:17 -0800 Subject: [PATCH 45/52] better names --- webapp/src/workspaces/workspace.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/webapp/src/workspaces/workspace.ts b/webapp/src/workspaces/workspace.ts index e1816149846..c85beb683f5 100644 --- a/webapp/src/workspaces/workspace.ts +++ b/webapp/src/workspaces/workspace.ts @@ -203,7 +203,7 @@ export function setupWorkspace(kind: WorkspaceKind): void { const msPerMin = 1000 * 60 const afterSync = (changed: Header[]) => { console.log(`...changes synced! ${!!changed}`) - onExternalHeaderChanges(changed) + onExternalChangesToHeaders(changed) } const doSync = async () => { console.log("synchronizing with the cloud..."); @@ -1596,17 +1596,17 @@ export async function saveToCloudAsync(h: Header) { // called when external changes happen to our headers (e.g. multi-tab // scenarios, cloud sync, etc.) -function onExternalHeaderChanges(changedHdrs: Header[]) { - changedHdrs.forEach(hd => { +function onExternalChangesToHeaders(newHdrs: Header[]) { + newHdrs.forEach(hd => { data.invalidateHeader("header", hd); data.invalidateHeader("text", hd); data.invalidateHeader("pkg-git-status", hd); }) - if (changedHdrs.length) { + if (newHdrs.length) { // TODO @darzu: can we make this more fine grain? data.invalidate("gh-commits:*"); // invalidate commits just in case console.log(`onExternalHeaderChanges:`) - console.dir(changedHdrs.map(toDbg)) // TODO @darzu: dbg + console.dir(newHdrs.map(toDbg)) // TODO @darzu: dbg } } @@ -1620,7 +1620,7 @@ export async function syncAsync(): Promise { const changedHdrs = await impl.synchronize({ expectedHeadersHash, }) - onExternalHeaderChanges(changedHdrs); + onExternalChangesToHeaders(changedHdrs); pxt.storage.setLocal('workspacesessionid', impl.getHeadersHash()); // TODO @darzu: handle: // filters?: pxt.editor.ProjectFilters; From bb676aa79b85faee2d0da3bd4e8dd085ab2160ca Mon Sep 17 00:00:00 2001 From: darzu Date: Sun, 3 Jan 2021 11:19:09 -0800 Subject: [PATCH 46/52] unify and fix up maybeSyncHeadersAsync, refreshHeadersSession and syncAsync --- webapp/src/package.ts | 3 + webapp/src/workspaces/cloudsyncworkspace.ts | 6 +- webapp/src/workspaces/workspace.ts | 134 +++++++++++++------- 3 files changed, 97 insertions(+), 46 deletions(-) diff --git a/webapp/src/package.ts b/webapp/src/package.ts index d7325932866..3626fd54c7f 100644 --- a/webapp/src/package.ts +++ b/webapp/src/package.ts @@ -405,6 +405,9 @@ export class EditorPackage { } savePkgAsync() { + console.log("savePkgAsync") // TODO @darzu: + return Promise.resolve() + // TODO @darzu: ensure none of this is needed; move or del if (this.header.cloudCurrent) return Promise.resolve(); this.savingNow++; this.updateStatus(); diff --git a/webapp/src/workspaces/cloudsyncworkspace.ts b/webapp/src/workspaces/cloudsyncworkspace.ts index 979d44d3f6f..c9ea123b701 100644 --- a/webapp/src/workspaces/cloudsyncworkspace.ts +++ b/webapp/src/workspaces/cloudsyncworkspace.ts @@ -113,7 +113,8 @@ export function createCachedWorkspace(ws: WorkspaceProvider): CachedWorkspacePro .filter(id => hasChanged(oldHdrsMap[id], newHdrsMap[id])) const newProjs = oldProjs // TODO @darzu: is there any point in copying here? for (let id of changedHdrIds) { - console.log(`cache invalidating ${id}`) // TODO @darzu: dbg + console.log(`cache invalidating ${id} because:`) // TODO @darzu: dbg + console.dir({ old: (oldHdrsMap[id]), new: (newHdrsMap[id]) }) delete newProjs[id] } @@ -210,7 +211,7 @@ export function createCachedWorkspace(ws: WorkspaceProvider): CachedWorkspacePro // TODO @darzu: dbging helper export function toDbg(h: Header) { - return {n: h.name, t: h.modificationTime, id: h.id} + return {n: h.name, t: h.modificationTime, del: h.isDeleted, id: h.id} } export interface CloudSyncWorkspace extends CachedWorkspaceProvider { @@ -277,6 +278,7 @@ export function createCloudSyncWorkspace(cloud: WorkspaceProvider, cloudLocal: W } async function synchronizeInternal(reason: SynchronizationReason): Promise { console.log("cloudsyncworkspace: synchronizeInternal") + console.dir(reason) // TODO @darzu: review these cases: // case 1: everything should be synced up, we're just polling the server diff --git a/webapp/src/workspaces/workspace.ts b/webapp/src/workspaces/workspace.ts index c85beb683f5..c520cb31481 100644 --- a/webapp/src/workspaces/workspace.ts +++ b/webapp/src/workspaces/workspace.ts @@ -371,31 +371,91 @@ export function getHeader(id: string) { return null } +// TODO @darzu: about workspacesessionid +/* + This represents the last known version of the headers + Any mutation should update the session hash. + The individual mutation will fail if we're out of sync + After we update, we should see if it is what we expected, if not, do a sync + We should regularly poll to see if there have been external changes. + Strategic points can check to see if there have been external changes. + + if we detect an external change, do sync: +*/ // this key is the max modificationTime value of the allHeaders // it is used to track if allHeaders need to be refreshed (syncAsync) let _allHeadersSessionHash: string = ""; // TODO @darzu: delete this (unneeded) +// useful because it is synchronous even though we always do the same thing if it is out of date export function isHeadersSessionOutdated() { return pxt.storage.getLocal('workspacesessionid') != _allHeadersSessionHash; } +// careful! only set the headers session after you know we were in sync before the mutation. +async function refreshHeadersSessionAfterMutation() { + await syncAsync() + const newHash = impl.getHeadersHash() + if (_allHeadersSessionHash !== newHash) { + _allHeadersSessionHash = newHash; + pxt.storage.setLocal('workspacesessionid', newHash); + } +} // TODO @darzu: delete this (unneeded) -function maybeSyncHeadersAsync(): Promise { - if (isHeadersSessionOutdated()) // another tab took control - return syncAsync().then(() => { }) // take back control - return Promise.resolve(); +async function maybeSyncHeadersAsync() { + return syncAsync() + // if (isHeadersSessionOutdated()) { // another tab made changes + // return syncAsync() // ensure we know what those changes were + // } + // return Promise.resolve(); } // TODO @darzu: delete this (unused) -function refreshHeadersSession() { - // use # of scripts + time of last mod as key - _allHeadersSessionHash = impl.getHeadersHash(); - - if (isHeadersSessionOutdated()) { - pxt.storage.setLocal('workspacesessionid', _allHeadersSessionHash); - pxt.debug(`workspace: refreshed headers session to ${_allHeadersSessionHash}`); - data.invalidate("header:*"); - data.invalidate("text:*"); +async function refreshHeadersSession() { + return await syncAsync() + + // TODO @darzu: del + // // use # of scripts + time of last mod as key + // _allHeadersSessionHash = impl.getHeadersHash(); + + // if (isHeadersSessionOutdated()) { + // pxt.storage.setLocal('workspacesessionid', _allHeadersSessionHash); + // pxt.debug(`workspace: refreshed headers session to ${_allHeadersSessionHash}`); + // console.log(`workspace: refreshed headers session to ${_allHeadersSessionHash}`); // TODO @darzu: dbg + // data.invalidate("header:*"); + // data.invalidate("text:*"); + // } +} +// contract post condition: the headers session will be up to date +export async function syncAsync(): Promise { + console.log("workspace:syncAsync"); + // TODO @darzu: ... and re-acquires headers ? + // TODO @darzu: clean up naming, layering + const expectedHeadersHash = pxt.storage.getLocal('workspacesessionid') + if (expectedHeadersHash !== _allHeadersSessionHash) { + const changedHdrs = await impl.synchronize({ + expectedHeadersHash, + }) + const newHash = impl.getHeadersHash() + _allHeadersSessionHash = newHash; + pxt.storage.setLocal('workspacesessionid', newHash); + onExternalChangesToHeaders(changedHdrs); } + // TODO @darzu: handle: + // filters?: pxt.editor.ProjectFilters; + // searchBar?: boolean; + + // TODO @darzu: \/ + /* + // force reload + ex.text = undefined + ex.version = undefined + + + impl.getSyncState() + */ + return {} } + +// TODO @darzu: check the usage of these three... we need to be really disciplined and ensure this fits +// with the all headers session hash usage. // this is an identifier for the current frame // in order to lock headers for editing const workspaceID: string = pxt.Util.guidGen(); @@ -540,6 +600,8 @@ export async function saveAsync(header: Header, text?: ScriptText, isCloud?: boo console.log("BAD double save!") // TODO @darzu: dbg // TODO @darzu: // return + }if (!text) { + console.log("BAD blank save!") } } @@ -559,6 +621,8 @@ export async function saveAsync(header: Header, text?: ScriptText, isCloud?: boo // await impl.setAsync(header, prj.version, text); } + await refreshHeadersSessionAfterMutation(); + return; } @@ -605,7 +669,7 @@ export function saveAsync2(h: Header, text?: ScriptText, isCloud?: boolean): Pro return headerQ.enqueue(h.id, () => fixupVersionAsync(e).then(() => impl.deleteAsync ? impl.deleteAsync(h, e.version) : impl.setAsync(h, e.version, {}))) - .finally(() => refreshHeadersSession()) + .finally(() => refreshHeadersSessionAfterMutation()) } // check if we have dynamic boards, store board info for home page rendering @@ -642,12 +706,12 @@ export function saveAsync2(h: Header, text?: ScriptText, isCloud?: boolean): Pro h.pubCurrent = false; h.blobCurrent_ = false; h.saveId = null; - // TODO @darzu: more data api syncing.. + // TODO @darzu: we shouldn't need these invalidates; double check data.invalidate("text:" + h.id); data.invalidate("pkg-git-status:" + h.id); } - refreshHeadersSession(); + refreshHeadersSessionAfterMutation(); }); } @@ -890,6 +954,7 @@ export function bumpedVersion(cfg: pxt.PackageConfig) { } export async function bumpAsync(hd: Header, newVer = "") { + console.log("bumpAsync") // TODO @darzu: dbg checkHeaderSession(hd); let files = await getTextAsync(hd.id) @@ -1296,6 +1361,9 @@ export async function exportToGithubAsync(hd: Header, repoid: string) { export async function recomputeHeaderFlagsAsync(h: Header, files: ScriptText) { checkHeaderSession(h); + // TODO @darzu: dbg + console.log("recomputeHeaderFlagsAsync") + h.githubCurrent = false const gitjson: GitJson = JSON.parse(files[GIT_JSON] || "{}") @@ -1418,7 +1486,11 @@ export function prepareConfigForGithub(content: string, createRelease?: boolean) } } -export async function initializeGithubRepoAsync(hd: Header, repoid: string, forceTemplateFiles: boolean, binaryJs: boolean) { +export async function initializeGithubRepoAsync(hd: Header, repoid: string, forceTemplateFiles: boolean, bAinaryJs: boolean) { + // TODO @darzu: dbg + console.log("initializeGithubRepoAsync") + // TODO @darzu: understand this function + await cloudsync.ensureGitHubTokenAsync(); let parsed = pxt.github.parseRepoId(repoid) @@ -1580,6 +1652,7 @@ export function installByIdAsync(id: string) { }, files))) } +// TODO @darzu: no one should call this export async function saveToCloudAsync(h: Header) { checkHeaderSession(h); if (!await auth.loggedIn()) @@ -1610,33 +1683,6 @@ function onExternalChangesToHeaders(newHdrs: Header[]) { } } -export async function syncAsync(): Promise { - console.log("workspace:syncAsync"); - // contract: - // output: this tab's headers session is up to date ... - // TODO @darzu: ... and re-acquires headers ? - // TODO @darzu: clean up naming, layering - const expectedHeadersHash = pxt.storage.getLocal('workspacesessionid') - const changedHdrs = await impl.synchronize({ - expectedHeadersHash, - }) - onExternalChangesToHeaders(changedHdrs); - pxt.storage.setLocal('workspacesessionid', impl.getHeadersHash()); - // TODO @darzu: handle: - // filters?: pxt.editor.ProjectFilters; - // searchBar?: boolean; - - // TODO @darzu: \/ - /* - // force reload - ex.text = undefined - ex.version = undefined - - - impl.getSyncState() - */ - return {} -} // this promise is set while a sync is in progress // cleared when sync is done. From a62252d5c0de42251ad663144eda6f0228c60a2d Mon Sep 17 00:00:00 2001 From: darzu Date: Sun, 3 Jan 2021 11:32:53 -0800 Subject: [PATCH 47/52] undo typo --- webapp/src/workspaces/workspace.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp/src/workspaces/workspace.ts b/webapp/src/workspaces/workspace.ts index c520cb31481..8f218308101 100644 --- a/webapp/src/workspaces/workspace.ts +++ b/webapp/src/workspaces/workspace.ts @@ -1486,7 +1486,7 @@ export function prepareConfigForGithub(content: string, createRelease?: boolean) } } -export async function initializeGithubRepoAsync(hd: Header, repoid: string, forceTemplateFiles: boolean, bAinaryJs: boolean) { +export async function initializeGithubRepoAsync(hd: Header, repoid: string, forceTemplateFiles: boolean, binaryJs: boolean) { // TODO @darzu: dbg console.log("initializeGithubRepoAsync") // TODO @darzu: understand this function From 1d523197380187aa80041bb8a23ac16eb552ce5b Mon Sep 17 00:00:00 2001 From: darzu Date: Sun, 3 Jan 2021 19:20:51 -0800 Subject: [PATCH 48/52] debugging spurious saves --- webapp/src/package.ts | 4 +- webapp/src/workspaces/cloudsyncworkspace.ts | 8 ++- webapp/src/workspaces/workspace.ts | 73 ++++++++++++++++++--- 3 files changed, 72 insertions(+), 13 deletions(-) diff --git a/webapp/src/package.ts b/webapp/src/package.ts index 3626fd54c7f..3a540bbaafd 100644 --- a/webapp/src/package.ts +++ b/webapp/src/package.ts @@ -629,6 +629,8 @@ class Host } downloadPackageAsync(pkg: pxt.Package): Promise { + console.log("downloadPackageAsync") // TODO @darzu: dbg + // TODO @darzu: what is the package abstraction for and why is it different than a workspace? let proto = pkg.verProtocol() let epkg = getEditorPkg(pkg) @@ -643,7 +645,7 @@ class Host // TODO @darzu: this is happening. return Promise.reject(new Error(`Cannot find text for package '${arg}' in the workspace.`)); if (epkg.isTopLevel() && epkg.header) - return workspace.recomputeHeaderFlagsAsync(epkg.header, scr) + return workspace.recomputeHeaderGitFlagsAsync(epkg.header, scr) .then(() => epkg.setFiles(scr)) else { epkg.setFiles(scr) diff --git a/webapp/src/workspaces/cloudsyncworkspace.ts b/webapp/src/workspaces/cloudsyncworkspace.ts index c9ea123b701..56914f7ac08 100644 --- a/webapp/src/workspaces/cloudsyncworkspace.ts +++ b/webapp/src/workspaces/cloudsyncworkspace.ts @@ -113,9 +113,11 @@ export function createCachedWorkspace(ws: WorkspaceProvider): CachedWorkspacePro .filter(id => hasChanged(oldHdrsMap[id], newHdrsMap[id])) const newProjs = oldProjs // TODO @darzu: is there any point in copying here? for (let id of changedHdrIds) { - console.log(`cache invalidating ${id} because:`) // TODO @darzu: dbg - console.dir({ old: (oldHdrsMap[id]), new: (newHdrsMap[id]) }) - delete newProjs[id] + if (id in newProjs) { + console.log(`cache invalidating ${id} because:`) // TODO @darzu: dbg + console.dir({ old: (oldHdrsMap[id]), new: (newHdrsMap[id]) }) + delete newProjs[id] + } } // save results diff --git a/webapp/src/workspaces/workspace.ts b/webapp/src/workspaces/workspace.ts index 8f218308101..6e97c1daff8 100644 --- a/webapp/src/workspaces/workspace.ts +++ b/webapp/src/workspaces/workspace.ts @@ -202,7 +202,7 @@ export function setupWorkspace(kind: WorkspaceKind): void { // TODO @darzu: improve this const msPerMin = 1000 * 60 const afterSync = (changed: Header[]) => { - console.log(`...changes synced! ${!!changed}`) + console.log(`...changes synced! # of changes ${changed.length}`) onExternalChangesToHeaders(changed) } const doSync = async () => { @@ -582,20 +582,74 @@ export function forceSaveAsync(h: Header, text?: ScriptText, isCloud?: boolean): return saveAsync(h, text, isCloud); } +// TODO @darzu: for debugging +function computeDiff(a: {header: Header, text: ScriptText}, b: {header: Header, text: ScriptText}): string { + const indent = (s: string) => '\t' + s + let res = '' + + if (!a.header || !a.text || !b.header || !b.text) { + res += `FULL: a.header:${!!a.header}, a.text:${!!a.text}, b.header:${!!b.header}, b.text:${!!b.text}` + return res; + } + + // headers + type HeaderK = keyof Header + const hdrKeys = U.unique([...Object.keys(a.header), ...Object.keys(b.header)], s => s) as HeaderK[] + const hasObjChanged = (a: any, b: any) => JSON.stringify(a) !== JSON.stringify(b) + const hasHdrChanged = (k: HeaderK) => hasObjChanged(a.header[k], b.header[k]) + const hdrChanges = hdrKeys.filter(hasHdrChanged) + const hdrDels = hdrChanges.filter(k => (k in a.header) && !(k in b.header)) + const hdrAdds = hdrChanges.filter(k => !(k in a.header) && (k in b.header)) + const hdrMods = hdrChanges.filter(k => (k in a.header) && (k in b.header)) + + res += `HEADER (+${hdrAdds.length}-${hdrDels.length}~${hdrMods.length})` + res += '\n' + const hdrDelStrs = hdrDels.map(k => `DEL ${k}`) + const hdrAddStrs = hdrAdds.map(k => `ADD ${k}: ${JSON.stringify(b.header[k])}`) + const hdrModStrs = hdrMods.map(k => `MOD ${k}: ${JSON.stringify(a.header[k])} => ${JSON.stringify(b.header[k])}`) + res += [...hdrDelStrs, ...hdrAddStrs, ...hdrModStrs].map(indent).join("\n") + res += '\n' + + // files + const filenames = U.unique([...Object.keys(a.text ?? {}), ...Object.keys(b.text ?? {})], s => s) + const hasFileChanged = (filename: string) => a.text[filename] !== b.text[filename] + const fileChanges = filenames.filter(hasFileChanged) + const fileDels = fileChanges.filter(k => (k in a.text) && !(k in b.text)) + const fileAdds = fileChanges.filter(k => !(k in a.text) && (k in b.text)) + const fileMods = fileChanges.filter(k => (k in a.text) && (k in b.text)) + + res += `FILES (+${fileAdds.length}-${fileDels.length}~${fileMods.length})` + res += '\n' + const fileDelStrs = fileDels.map(k => `DEL ${k}`) + const fileAddStrs = fileAdds.map(k => `ADD ${k}`) + const fileModStrs = fileMods.map(k => `MOD ${k}: ${a.text[k].length} => ${b.text[k].length}`) + res += [...fileDelStrs, ...fileAddStrs, ...fileModStrs].map(indent).join("\n") + res += '\n' + + return res; +} + export async function saveAsync(header: Header, text?: ScriptText, isCloud?: boolean): Promise { console.log(`workspace:saveAsync ${header.id}`) // TODO @darzu: port over from saveAsync2 - let prj = await impl.getAsync(header) // TODO @darzu: dbg - if (!prj) { + let prevProj = await impl.getAsync(header) // TODO @darzu: dbg + if (!prevProj) { // first save console.log(`first save: ${header.id}`) // TODO @darzu: dbg - prj = { + prevProj = { header, text, version: null } } else { - if (prj.text === text) { + const diff = computeDiff(prevProj, { + header, + text, + }) + console.log(`changes to ${header.id}:`) + console.log(diff) + + if (prevProj.text === text) { // we're getting a double save.. no point console.log("BAD double save!") // TODO @darzu: dbg // TODO @darzu: @@ -606,10 +660,10 @@ export async function saveAsync(header: Header, text?: ScriptText, isCloud?: boo } try { - const res = await impl.setAsync(header, prj.version, text); + const res = await impl.setAsync(header, prevProj.version, text); if (!res) { // conflict occured - console.log(`conflict occured for ${header.id} at ${prj.version}`) // TODO @darzu: dbg + console.log(`conflict occured for ${header.id} at ${prevProj.version}`) // TODO @darzu: dbg // TODO @darzu: what to do? probably nothing } } catch (e) { @@ -865,8 +919,9 @@ export async function hasPullAsync(hd: Header) { } export async function pullAsync(hd: Header, checkOnly = false) { + console.log("pullAsync") // TODO @darzu: dbg let files = await getTextAsync(hd.id) - await recomputeHeaderFlagsAsync(hd, files) + await recomputeHeaderGitFlagsAsync(hd, files) let gitjsontext = files[GIT_JSON] if (!gitjsontext) return PullStatus.NoSourceControl @@ -1358,7 +1413,7 @@ export async function exportToGithubAsync(hd: Header, repoid: string) { // to be called after loading header in a editor -export async function recomputeHeaderFlagsAsync(h: Header, files: ScriptText) { +export async function recomputeHeaderGitFlagsAsync(h: Header, files: ScriptText) { checkHeaderSession(h); // TODO @darzu: dbg From a3155bde10e60d49a5ff7c72606ea70cb9118339 Mon Sep 17 00:00:00 2001 From: darzu Date: Sun, 3 Jan 2021 19:25:45 -0800 Subject: [PATCH 49/52] first-save TODOs --- webapp/src/workspaces/workspace.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/webapp/src/workspaces/workspace.ts b/webapp/src/workspaces/workspace.ts index 6e97c1daff8..faac4392610 100644 --- a/webapp/src/workspaces/workspace.ts +++ b/webapp/src/workspaces/workspace.ts @@ -70,6 +70,8 @@ type File = pxt.workspace.File; // [ ] conflict resolution dialog // [ ] in editor // [ ] refactor out git / github stuff +// [ ] on first save: investigate conflicting save +// [ ] on first save: investigate NULL save // [ ] clean up code // [ ] handle all TODO @darzu's // [ ] renames From d70f46a00490e6bc96a259b1dd0bdd31dfdba5ac Mon Sep 17 00:00:00 2001 From: darzu Date: Sun, 3 Jan 2021 19:38:42 -0800 Subject: [PATCH 50/52] debugging empty saves --- webapp/src/app.tsx | 1 + webapp/src/workspaces/workspace.ts | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/webapp/src/app.tsx b/webapp/src/app.tsx index 0c986a576fc..3022822b641 100644 --- a/webapp/src/app.tsx +++ b/webapp/src/app.tsx @@ -1451,6 +1451,7 @@ export class ProjectView } // update recentUse on the header + // TODO @darzu: this is saving hte project without text... return workspace.saveAsync(h) }).then(() => this.loadTutorialFiltersAsync()) .finally(() => { diff --git a/webapp/src/workspaces/workspace.ts b/webapp/src/workspaces/workspace.ts index faac4392610..4f0a207f048 100644 --- a/webapp/src/workspaces/workspace.ts +++ b/webapp/src/workspaces/workspace.ts @@ -633,6 +633,12 @@ function computeDiff(a: {header: Header, text: ScriptText}, b: {header: Header, export async function saveAsync(header: Header, text?: ScriptText, isCloud?: boolean): Promise { console.log(`workspace:saveAsync ${header.id}`) + if (!text) { + console.log("BAD blank save!") + // debugger; // TODO @darzu: dbg + // TODO @darzu: just return. that's what old browser workspace and saveAsync2 do in combo + } + // TODO @darzu: port over from saveAsync2 let prevProj = await impl.getAsync(header) // TODO @darzu: dbg if (!prevProj) { @@ -656,9 +662,8 @@ export async function saveAsync(header: Header, text?: ScriptText, isCloud?: boo console.log("BAD double save!") // TODO @darzu: dbg // TODO @darzu: // return - }if (!text) { - console.log("BAD blank save!") } + } try { From 0ea581364bb134aae70d286dffdaa4ac6030caa1 Mon Sep 17 00:00:00 2001 From: darzu Date: Mon, 4 Jan 2021 11:18:10 -0800 Subject: [PATCH 51/52] fix issues with getTextAsync and lookup(); investigating double save to cloud --- pxteditor/workspace.ts | 5 +- webapp/src/workspaces/browserdbworkspace.ts | 63 ++++---- webapp/src/workspaces/cloudsyncworkspace.ts | 30 ++-- webapp/src/workspaces/jointworkspace.ts | 10 +- webapp/src/workspaces/workspace.ts | 163 ++++++++++++++------ 5 files changed, 173 insertions(+), 98 deletions(-) diff --git a/pxteditor/workspace.ts b/pxteditor/workspace.ts index 48eab71cbed..88ebcda375e 100644 --- a/pxteditor/workspace.ts +++ b/pxteditor/workspace.ts @@ -29,12 +29,15 @@ namespace pxt.workspace { } export interface WorkspaceProvider { - listAsync(): Promise; // called from workspace.syncAsync (including upon startup) + listAsync(): Promise; /* Tries to get the corrisponding File with the current version if it exists. If it does not exist, returns undefined. */ getAsync(h: Header): Promise; + /* + If text is empty, then only update the header. + */ setAsync(h: Header, prevVersion: Version, text?: ScriptText): Promise; deleteAsync?: (h: Header, prevVersion: Version) => Promise; resetAsync(): Promise; diff --git a/webapp/src/workspaces/browserdbworkspace.ts b/webapp/src/workspaces/browserdbworkspace.ts index 8e0e66c1ab5..1a3e42882c6 100644 --- a/webapp/src/workspaces/browserdbworkspace.ts +++ b/webapp/src/workspaces/browserdbworkspace.ts @@ -47,42 +47,42 @@ export function createBrowserDbWorkspace(namespace: string): BrowserDbWorkspaceP return hdrs } async function getAsync(h: Header): Promise { - const resp: TextDbEntry = await textDb.getAsync(h.id) - if (!resp) - return undefined; + const hdrProm = headerDb.getAsync(h.id) + const textProm = textDb.getAsync(h.id) + let [hdrResp, textResp] = await Promise.all([hdrProm, textProm]) as [Header, TextDbEntry] + if (!hdrResp || !textResp) + // TODO @darzu: distinguish these for the caller somehow? + return undefined return { - header: h, - text: resp.files, - version: resp._rev + header: hdrResp, + text: textResp.files, + version: textResp._rev } } async function setAsync(h: Header, prevVer: any, text?: ScriptText): Promise { - // TODO @darzu: debug logging - if (!text) { - // TODO @darzu: trace down why... this is a real bug - console.log("!!! setAsync without text :(") - // console.dir(h) - } else { - console.log(`setAsync ${namespace || "default"}:(${h.id}, ${h.modificationTime}, ${prevVer}) :)`) - } - - const textEnt: TextDbEntry = { - files: text, - id: h.id, - _rev: prevVer - } + // TODO @darzu: dbg + console.log(`setAsync ${namespace || "default"}:(${h.id}, ${h.modificationTime}, ${prevVer}) :)`) + + let textVer: string = undefined; + if (text) { + const textEnt: TextDbEntry = { + files: text, + id: h.id, + _rev: prevVer + } - // if we get a 400, we need to fetch the old then do a new - let textVer: string; - try { - textVer = await textDb.setAsync(textEnt) - } catch (e) {} + // if we get a 400, we need to fetch the old then do a new + // TODO @darzu: no we shouldn't; this isn't the right layer to handle storage conflicts + try { + textVer = await textDb.setAsync(textEnt) + } catch (e) {} - if (!textVer) { - console.log(`! failed to set text for id:${h.id},pv:${prevVer}`); // TODO @darzu: dbg logging - const oldTxt = await textDb.getAsync(h.id) - console.dir(`! text ${h.id} actually is: ${oldTxt._rev}`) - console.dir(oldTxt) + if (!textVer) { + console.log(`! failed to set text for id:${h.id},pv:${prevVer}`); // TODO @darzu: dbg logging + const oldTxt = await textDb.getAsync(h.id) + console.dir(`! text ${h.id} actually is: ${oldTxt._rev}`) + console.dir(oldTxt) + } } let hdrVer: string; @@ -102,6 +102,7 @@ export function createBrowserDbWorkspace(namespace: string): BrowserDbWorkspaceP delete h._rev } // TODO @darzu: need to rethink error handling here + // TODO @darzu: we shouldn't auto-retry on conflict failure try { hdrVer = await headerDb.setAsync(h) } catch (e) {} @@ -109,11 +110,11 @@ export function createBrowserDbWorkspace(namespace: string): BrowserDbWorkspaceP console.log(`!!! failed AGAIN to set hdr for id:${h.id},old:${JSON.stringify(oldHdr)}`); // TODO @darzu: dbg logging } } - h._rev = hdrVer await printDbg(); // TODO @darzu: dbg + // TODO @darzu: notice undefined means either: "version conflict when setting text" and "no text sent" return textVer } async function deleteAsync(h: Header, prevVer: any): Promise { diff --git a/webapp/src/workspaces/cloudsyncworkspace.ts b/webapp/src/workspaces/cloudsyncworkspace.ts index 56914f7ac08..9ac70a3cdce 100644 --- a/webapp/src/workspaces/cloudsyncworkspace.ts +++ b/webapp/src/workspaces/cloudsyncworkspace.ts @@ -19,7 +19,7 @@ export interface CachedWorkspaceProvider extends WorkspaceProvider { pendingSync(): Promise, firstSync(): Promise, listSync(): Header[], - hasSync(h: Header): boolean, + getHeaderSync(id: string): Header, tryGetSync(h: Header): WsFile } @@ -145,11 +145,13 @@ export function createCachedWorkspace(ws: WorkspaceProvider): CachedWorkspacePro } async function setAsync(h: Header, prevVer: Version, text?: ScriptText): Promise { await pendingUpdate; - // update cached projects - cacheProjs[h.id] = { - header: h, - text, - version: prevVer + if (text) { + // update cached projects + cacheProjs[h.id] = { + header: h, + text, + version: prevVer + } } // update headers list, map and hash if (!cacheHdrsMap[h.id]) { @@ -160,11 +162,13 @@ export function createCachedWorkspace(ws: WorkspaceProvider): CachedWorkspacePro // send update to backing storage const res = await ws.setAsync(h, prevVer, text) if (res) { - // update cached projec - cacheProjs[h.id] = { - header: h, - text, - version: res + if (text) { + // update cached project + cacheProjs[h.id] = { + header: h, + text, + version: res + } } } else { // conflict; delete cache @@ -198,8 +202,8 @@ export function createCachedWorkspace(ws: WorkspaceProvider): CachedWorkspacePro pendingSync: () => pendingUpdate, firstSync: () => firstUpdate, listSync: () => cacheHdrs, - hasSync: h => h && !!cacheHdrsMap[h.id], tryGetSync: h => cacheProjs[h.id], + getHeaderSync: id => cacheHdrsMap[id], // workspace getAsync, setAsync, @@ -395,8 +399,8 @@ export function createCloudSyncWorkspace(cloud: WorkspaceProvider, cloudLocal: W pendingSync: () => pendingSync, firstSync: () => firstSync, listSync: () => localCache.listSync(), - hasSync: h => localCache.hasSync(h), tryGetSync: h => localCache.tryGetSync(h), + getHeaderSync: id => localCache.getHeaderSync(id), // workspace getAsync, setAsync, diff --git a/webapp/src/workspaces/jointworkspace.ts b/webapp/src/workspaces/jointworkspace.ts index 40b57bcc0ca..3499cfadce1 100644 --- a/webapp/src/workspaces/jointworkspace.ts +++ b/webapp/src/workspaces/jointworkspace.ts @@ -103,7 +103,7 @@ export function createJointWorkspace(...all: CachedWorkspaceProvider[]): CachedW .reduce((p, n) => [...p, ...n], []) } function getWorkspaceFor(h: Header): CachedWorkspaceProvider { - return all.reduce((p, n) => p || (n.hasSync(h) ? n : null), null) + return all.reduce((p, n) => p || (n.getHeaderSync(h?.id) ? n : null), null) } async function getAsync(h: Header): Promise { await pendingSync() @@ -116,14 +116,14 @@ export function createJointWorkspace(...all: CachedWorkspaceProvider[]): CachedW const ws = getWorkspaceFor(h) return ws?.tryGetSync(h) ?? undefined } - function hasSync(h: Header): boolean { - return all.reduce((p, n) => p || n.hasSync(h), false) + function getHeaderSync(id: string): Header { + return all.reduce((p, n) => p || n.getHeaderSync(id), null as Header) } async function setAsync(h: Header, prevVer: any, text?: ScriptText): Promise { await pendingSync() // TODO @darzu: dbg logging console.log("joint:setAsync") - console.dir(all.map(w => w.hasSync(h))) + console.dir(all.map(w => w.getHeaderSync(h.id))) const ws = getWorkspaceFor(h) ?? all[0] return ws.setAsync(h, prevVer, text) } @@ -144,8 +144,8 @@ export function createJointWorkspace(...all: CachedWorkspaceProvider[]): CachedW pendingSync, firstSync, listSync, - hasSync, tryGetSync, + getHeaderSync, // workspace getAsync, setAsync, diff --git a/webapp/src/workspaces/workspace.ts b/webapp/src/workspaces/workspace.ts index 4f0a207f048..29e26d6d138 100644 --- a/webapp/src/workspaces/workspace.ts +++ b/webapp/src/workspaces/workspace.ts @@ -107,9 +107,18 @@ let implType: WorkspaceKind; // TODO @darzu: del function lookup(id: string): File { + if (!id) { + console.log(`! looking up null id`) // TODO @darzu: dbg + } + // TODO @darzu: what is .path used for? - const hdr = impl.listSync().find(h => h.id == id || h.path == id) - return impl.tryGetSync(hdr) + const hdr = impl.getHeaderSync(id) + const resp = impl.tryGetSync(hdr) + + if (!resp) { + console.log(`! lookup for ${id} failed!`) // TODO @darzu: dbg + } + return resp // TODO @darzu: del // implCache.getSync(); // return allScripts.find(x => x.header.id == id || x.header.path == id); @@ -365,13 +374,22 @@ function cleanupBackupsAsync() { })); } -export function getHeader(id: string) { - maybeSyncHeadersAsync().done(); - let e = lookup(id) - if (e && !e.header.isDeleted) - return e.header +export function getHeader(id: string): Header { + maybeSyncHeadersAsync().done(); // TODO @darzu: handle properly + const hdr = impl.getHeaderSync(id) + if (hdr && !hdr.isDeleted) // TODO @darzu: ensure we're treating soft delete consistently + return hdr + console.log(`! cannot find header: ${id}`) // TODO @darzu: dbg return null } +// TODO @darzu: delete +// export function getHeader2(id: string) { +// maybeSyncHeadersAsync().done(); +// let e = lookup(id) +// if (e && !e.header.isDeleted) +// return e.header +// return null +// } // TODO @darzu: about workspacesessionid /* @@ -499,25 +517,60 @@ export function initAsync() { }) } -export function getTextAsync(id: string): Promise { - return maybeSyncHeadersAsync() - .then(() => { - let e = lookup(id) - if (!e) - return Promise.resolve(null as ScriptText) - if (e.text) - return Promise.resolve(e.text) - return headerQ.enqueue(id, () => impl.getAsync(e.header) - .then(resp => { - if (!e.text) { - // otherwise we were beaten to it - e.text = fixupFileNames(resp.text); - } - e.version = resp.version; - return e.text - })) - }) -} +export async function getTextAsync(id: string): Promise { + await maybeSyncHeadersAsync(); + const hdr = impl.getHeaderSync(id); + if (!hdr) { + console.log(`! Lookup failed for ${id}`); // TODO @darzu: dbg + return null + } + const proj = await impl.getAsync(hdr) + if (!proj) { + // TODO @darzu: this is a bad scenario. we should probably purge the header + console.log(`!!! FOUND HEADER BUT NOT PROJECT TEXT FOR: ${id}`); // TODO @darzu: dbg + console.dir(hdr) + return null + } + return proj.text + // TODO @darzu: incorperate: + // return + // .then(() => { + // let e = lookup(id) + // if (!e) + // return Promise.resolve(null as ScriptText) + // if (e.text) + // return Promise.resolve(e.text) + // return headerQ.enqueue(id, () => impl.getAsync(e.header) + // .then(resp => { + // if (!e.text) { + // // otherwise we were beaten to it + // e.text = fixupFileNames(resp.text); + // } + // e.version = resp.version; + // return e.text + // })) + // }) +} +// TODO @darzu: delete +// export function getTextAsync2(id: string): Promise { +// return maybeSyncHeadersAsync() +// .then(() => { +// let e = lookup(id) +// if (!e) +// return Promise.resolve(null as ScriptText) +// if (e.text) +// return Promise.resolve(e.text) +// return headerQ.enqueue(id, () => impl.getAsync(e.header) +// .then(resp => { +// if (!e.text) { +// // otherwise we were beaten to it +// e.text = fixupFileNames(resp.text); +// } +// e.version = resp.version; +// return e.text +// })) +// }) +// } export interface ScriptMeta { description: string; @@ -640,37 +693,51 @@ export async function saveAsync(header: Header, text?: ScriptText, isCloud?: boo } // TODO @darzu: port over from saveAsync2 - let prevProj = await impl.getAsync(header) // TODO @darzu: dbg - if (!prevProj) { - // first save - console.log(`first save: ${header.id}`) // TODO @darzu: dbg - prevProj = { - header, - text, - version: null + let newProj: File; + if (text) { + // header & text insert/update + let prevProj = await impl.getAsync(header) // TODO @darzu: dbg + if (prevProj) { + // update + newProj = { + ...prevProj, + header, + text + } + } else { + // new project + newProj = { + header, + text, + version: null, + } + console.log(`first save: ${header.id}`) // TODO @darzu: dbg + } + + // TODO @darzu: dbg + if (prevProj) { + // TODO @darzu: dbg: + const diff = computeDiff(prevProj, { + header, + text, + }) + console.log(`changes to ${header.id}:`) + console.log(diff) } } else { - const diff = computeDiff(prevProj, { + // header only update + newProj = { header, - text, - }) - console.log(`changes to ${header.id}:`) - console.log(diff) - - if (prevProj.text === text) { - // we're getting a double save.. no point - console.log("BAD double save!") // TODO @darzu: dbg - // TODO @darzu: - // return + text: null, + version: null, } - } try { - const res = await impl.setAsync(header, prevProj.version, text); + const res = await impl.setAsync(newProj.header, newProj.version, newProj.text); if (!res) { // conflict occured - console.log(`conflict occured for ${header.id} at ${prevProj.version}`) // TODO @darzu: dbg + console.log(`conflict occured for ${header.id} at ${newProj.version}`) // TODO @darzu: dbg // TODO @darzu: what to do? probably nothing } } catch (e) { From 1e93dd1959735c2df9dcb3f268a88dcfa0115552 Mon Sep 17 00:00:00 2001 From: darzu Date: Mon, 4 Jan 2021 13:55:11 -0800 Subject: [PATCH 52/52] handling 404s from the cloud --- webapp/src/cloud.ts | 2 ++ webapp/src/workspaces/cloudsyncworkspace.ts | 8 +++++++- webapp/src/workspaces/workspace.ts | 11 ++++++++++- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/webapp/src/cloud.ts b/webapp/src/cloud.ts index ca5db194f5b..8f6ff44f202 100644 --- a/webapp/src/cloud.ts +++ b/webapp/src/cloud.ts @@ -66,6 +66,8 @@ export function getAsync(h: Header): Promise { file.header.cloudVersion = file.version; file.header.cloudUserId = userId; resolve(file); + } else if (result.statusCode === 404) { + resolve(null); } else { reject(new Error(result.errmsg)); } diff --git a/webapp/src/workspaces/cloudsyncworkspace.ts b/webapp/src/workspaces/cloudsyncworkspace.ts index 9ac70a3cdce..15e4dac66fd 100644 --- a/webapp/src/workspaces/cloudsyncworkspace.ts +++ b/webapp/src/workspaces/cloudsyncworkspace.ts @@ -366,9 +366,15 @@ export function createCloudSyncWorkspace(cloud: WorkspaceProvider, cloudLocal: W } async function setAsync(h: Header, prevVer: any, text?: ScriptText): Promise { await pendingSync + // TODO @darzu: cannot pass prevVer to both of these.. they have different meanings on the different platforms // TODO @darzu: use a queue to sync to backend and make sure this promise is part of the pending sync set - const cloudPromise = cloudCache.setAsync(h, prevVer, text) + async function cloudSet() { + const prevCloudProj = await cloudCache.getAsync(h) + const newCloudVer = await cloudCache.setAsync(h, prevCloudProj?.version, text) + } + cloudSet() + // TODO @darzu: also what to do with the return value ? return await localCache.setAsync(h, prevVer, text) } diff --git a/webapp/src/workspaces/workspace.ts b/webapp/src/workspaces/workspace.ts index 29e26d6d138..2f9ca8b6409 100644 --- a/webapp/src/workspaces/workspace.ts +++ b/webapp/src/workspaces/workspace.ts @@ -35,6 +35,16 @@ type WorkspaceProvider = pxt.workspace.WorkspaceProvider; type InstallHeader = pxt.workspace.InstallHeader; type File = pxt.workspace.File; +// TODO @darzu: cloud specific: +/* +MIN BAR: +[ ] UI around sync time +[ ] keeping cloud work in the background +NICE TO HAVE: +[ ] UI conflict resolution +[ ] UI around "manual refresh" +*/ + // TODO @darzu: todo list: // [ ] remove / fix header session methods // [ ] refreshHeadersSession @@ -139,7 +149,6 @@ export function copyProjectToLegacyEditor(header: Header, majorVersion: number): return browserworkspace.copyProjectToLegacyEditor(header, majorVersion); } - export type WorkspaceKind = "browser" | "fs" | "file" | "mem" | "memory" | "iframe" | "uwp" | "idb" | "cloud"; function chooseWorkspace(kind: WorkspaceKind = "browser"): pxt.workspace.WorkspaceProvider {