Skip to content

Commit

Permalink
Committing checkpoint to revert; this approach is messy, outlined bet…
Browse files Browse the repository at this point in the history 
…ter potential approach in pxtlib/package.ts
  • Loading branch information
@jwunderl
jwunderl committed May 23, 2023
1 parent 1be0b72 commit 404d63a
Show file tree
Hide file tree
Showing 8 changed files with 130 additions and 35 deletions.
1 change: 1 addition & 0 deletions pxtcompiler/emitter/driver.ts
View file
Expand Up @@ -271,6 +271,7 @@ namespace ts.pxtc {

if (opts.ast || opts.forceEmit || res.diagnostics.length == 0) {
const binOutput = compileBinary(program, opts, res, entryPoint);

res.times["compilebinary"] = U.cpuUs() - emitStart
res.diagnostics = res.diagnostics.concat(patchUpDiagnostics(binOutput.diagnostics))
}
Expand Down
1 change: 1 addition & 0 deletions pxtcompiler/emitter/hexfile.ts
View file
Expand Up @@ -1083,6 +1083,7 @@ _stored_program: .hex ${res}
src = asmHeader(bin) + src
}
if (opts.embedBlob) {

bin.packedSource = packSource(opts.embedMeta, ts.pxtc.decodeBase64(opts.embedBlob))
// TODO more dynamic check for source size
if (!bin.target.noSourceInFlash && bin.packedSource.length < 40000) {
Expand Down
14 changes: 10 additions & 4 deletions pxtlib/github.ts
View file
Expand Up @@ -630,7 +630,7 @@ namespace pxt.github {
return { version, config };
}

export async function cacheProjectDependenciesAsync(cfg: pxt.PackageConfig): Promise<void> {
export async function cacheProjectDependenciesAsync(cfg: pxt.PackageConfig, packagedFiles?: pxt.Map<string>): Promise<void> {
const ghExtensions = Object.keys(cfg.dependencies)
?.filter(dep => isGithubId(cfg.dependencies[dep]));

Expand All @@ -641,9 +641,15 @@ namespace pxt.github {
ghExtensions.map(
async ext => {
const extSrc = cfg.dependencies[ext];
const ghPkg = await downloadPackageAsync(extSrc, pkgConfig);
if (!ghPkg) {
throw new Error(lf("Cannot load extension {0} from {1}", ext, extSrc));
let ghPkg: CachedPackage;
let caughtError: any;
try {
ghPkg = await downloadPackageAsync(extSrc, pkgConfig);
} catch (e) {
caughtError = e;
}
if ((!ghPkg || caughtError) && !packagedFiles?.[`${extSrc}-backup.json`]) {
throw caughtError || new Error(lf("Cannot load extension {0} from {1}", ext, extSrc));
}
}
)
Expand Down
2 changes: 1 addition & 1 deletion pxtlib/main.ts
View file
Expand Up @@ -439,7 +439,7 @@ namespace pxt {
export interface Host {
readFile(pkg: Package, filename: string, skipAdditionalFiles?: boolean): string;
writeFile(pkg: Package, filename: string, contents: string, force?: boolean): void;
downloadPackageAsync(pkg: Package, deps?: string[]): Promise<void>;
downloadPackageAsync(pkg: Package, deps?: string[]/**, fallback?: () => void **/): Promise<void>;
getHexInfoAsync(extInfo: pxtc.ExtensionInfo): Promise<pxtc.HexInfo>;
cacheStoreAsync(id: string, val: string): Promise<void>;
cacheGetAsync(id: string): Promise<string>; // null if not found
Expand Down
120 changes: 94 additions & 26 deletions pxtlib/package.ts
View file
Expand Up @@ -142,12 +142,35 @@ namespace pxt {
const proto = this.verProtocol()
let files: Map<string>;

if (proto == "pub") {
files = await Cloud.downloadScriptFilesAsync(this.verArgument())
} else if (proto == "github") {
const config = await pxt.packagesConfigAsync();
const resp = await pxt.github.downloadPackageAsync(this.verArgument(), config)
files = resp.files;
if (proto == "pub" || proto == "github") {
let caughtError: any;
try {
if (proto == "pub") {
files = await Cloud.downloadScriptFilesAsync(this.verArgument())
} else {
const config = await pxt.packagesConfigAsync();
const resp = await pxt.github.downloadPackageAsync(this.verArgument(), config)
files = resp.files;
}
} catch (e) {
caughtError = e ;
}

// attempt to grab from top level package if `${dep._verspec}-backup.json` defined
let p = this.parent;
while (p && p != <Package>this) {
const backupFile = p.readFile(`${this._verspec}-backup.json`);
if (backupFile) {
files = JSON.parse(backupFile);
break;
}
p = p.parent;
}

if (!files && caughtError) {
// no backup found, rethrow
throw caughtError || new Error("Could not download.");
}
} else if (proto == "embed") {
files = pxt.getEmbeddedScript(this.verArgument())
} else if (proto == "pkg") {
Expand Down Expand Up @@ -307,27 +330,70 @@ namespace pxt {
return Promise.resolve(v)
}

private downloadAsync() {
return this.resolveVersionAsync()
.then(verNo => {
if (this.invalid()) {
pxt.debug(`skip download of invalid package ${this.id}`);
return undefined;
}
if (!/^embed:/.test(verNo) && this.installedVersion == verNo)
return undefined;
pxt.debug('downloading ' + verNo)
return this.host().downloadPackageAsync(this)
.then(() => {
this.loadConfig();

if (this.isAssetPack()) {
this.writeAssetPackFiles();
}
pxt.debug(`installed ${this.id} /${verNo}`)
})
/**
* strip out at beginning of hex file import
* push them into hex, github cache if those values are not already there
* flag attached that says they're temporary / backup only
* treat the cache as expired by time
* keep trying to fetch new one but return it in place until network is available
**/
private async downloadAsync(): Promise<void> {
const verNo = await this.resolveVersionAsync();
if (this.invalid()) {
pxt.debug(`skip download of invalid package ${this.id}`);
return undefined;
}
if (!/^embed:/.test(verNo) && this.installedVersion == verNo)
return undefined;
pxt.debug('downloading ' + verNo)
await this.host().downloadPackageAsync(
this,
// undefined,
// () => {
// let p = this.parent;
// while (p && p != <Package>this) {
// const backupFile = p.readFile(`${this._verspec}-backup.json`);
// if (backupFile) {
// return JSON.parse(backupFile);
// }
// p = p.parent;
// }

// return undefined;
// }
);
// try {
// } catch (e) {
// let files: Map<string>;
// let p = this.parent;
// while (p && p != <Package>this) {
// const backupFile = p.readFile(`${this._verspec}-backup.json`);
// if (backupFile) {
// files = JSON.parse(backupFile);
// break;
// }
// p = p.parent;
// }

// if (files) {
// // this.writeFile(pxt.CONFIG_NAME, files[pxt.CONFIG_NAME])
// // for (const fname of Object.keys(files)) {
// // this.writeFile(fname, files[fname]);
// // }
// // this
// } else {
// // no backup found, rethrow
// throw e;
// }
// }

this.loadConfig();

if (this.isAssetPack()) {
this.writeAssetPackFiles();
}

})
pxt.debug(`installed ${this.id} /${verNo}`)
}

loadConfig() {
Expand Down Expand Up @@ -1340,6 +1406,8 @@ namespace pxt {
for (const dep of depsToPack) {
// todo; swap this over to just one json blob under files,
// not keyed off -backup.json, to make extracting / hiding more obvious. key goes in pxtlib/main.ts

// include c++ pxtc.HexInfo as well?
if (files[`${dep._verspec}-backup.json`])
continue;
const packed: Map<string> = {}
Expand Down
2 changes: 1 addition & 1 deletion webapp/src/app.tsx
View file
Expand Up @@ -2868,7 +2868,7 @@ export class ProjectView
}
files[pxt.CONFIG_NAME] = pxt.Package.stringifyConfig(cfg);

await pxt.github.cacheProjectDependenciesAsync(cfg);
await pxt.github.cacheProjectDependenciesAsync(cfg, files);

const hd = await workspace.installAsync({
name: cfg.name,
Expand Down
22 changes: 20 additions & 2 deletions webapp/src/package.ts
View file
Expand Up @@ -456,7 +456,13 @@ export class EditorPackage {
sortedFiles(): File[] {
let lst = Util.values(this.files)
if (!pxt.options.debug)
lst = lst.filter(f => f.name != pxt.github.GIT_JSON && f.name != pxt.SIMSTATE_JSON && f.name != pxt.SERIAL_EDITOR_FILE && f.name != pxt.PALETTES_FILE)
lst = lst.filter(f =>
f.name != pxt.github.GIT_JSON
&& f.name != pxt.SIMSTATE_JSON
&& f.name != pxt.SERIAL_EDITOR_FILE
&& f.name != pxt.PALETTES_FILE
&& !/(pub|github):.+-backup.json/.test(f.name)
);
lst.sort((a, b) => a.weight() - b.weight() || Util.strcmp(a.name, b.name))
return lst
}
Expand Down Expand Up @@ -701,7 +707,7 @@ class Host
.then(v => v.val, e => null)
}

downloadPackageAsync(pkg: pxt.Package): Promise<void> {
downloadPackageAsync(pkg: pxt.Package, deps?: string[]/**, fallback?: () => void**/): Promise<void> {
let proto = pkg.verProtocol()
let epkg = getEditorPkg(pkg)

Expand All @@ -723,13 +729,25 @@ class Host
}
})

const handleMissingNetwork = (e: any) => {
// const files = fallback();
const files = mainPkg.readFile(`${pkg._verspec}-backup.json`);
if (files) {
epkg.setFiles(JSON.parse(files));
} else {
throw e;
}
}

if (proto == "pub") {
// make sure it sits in cache
return workspace.getPublishedScriptAsync(pkg.verArgument())
.then(files => epkg.setFiles(files))
.catch(handleMissingNetwork);
} else if (proto == "github") {
return workspace.getPublishedScriptAsync(pkg.version())
.then(files => epkg.setFiles(files))
.catch(handleMissingNetwork);
} else if (proto == "workspace") {
return fromWorkspaceAsync(pkg.verArgument())
} else if (proto == "file") {
Expand Down
3 changes: 2 additions & 1 deletion webapp/src/workspace.ts
View file
Expand Up @@ -646,7 +646,8 @@ export function installAsync(h0: InstallHeader, text: ScriptText, dontOverwriteI
pxt.shell.setEditorLanguagePref(cfg.preferredEditor);
}

return pxt.github.cacheProjectDependenciesAsync(cfg)

return pxt.github.cacheProjectDependenciesAsync(cfg, text)
.then(() => importAsync(h, text))
.then(() => h);
}
Expand Down

0 comments on commit 404d63a

Please sign in to comment.