From 1be0b7269519df701cb49359577be309d58dbaac Mon Sep 17 00:00:00 2001 From: Joey Wunderlich Date: Tue, 23 May 2023 10:01:33 -0700 Subject: [PATCH 1/9] pack in external deps (pub:/github:) to generated hex files --- pxtlib/package.ts | 70 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 53 insertions(+), 17 deletions(-) diff --git a/pxtlib/package.ts b/pxtlib/package.ts index 30a55fc6b8a6..9ca53e06ccc1 100644 --- a/pxtlib/package.ts +++ b/pxtlib/package.ts @@ -1235,7 +1235,11 @@ namespace pxt { !opts.target.isNative if (!noFileEmbed) { - const files = await this.filesToBePublishedAsync(true) + // Include packages when it won't influence flash size. + const files = await this.filesToBePublishedAsync( + true, + !appTarget.compile.useUF2 + ); const headerString = JSON.stringify({ name: this.config.name, comment: this.config.description, @@ -1302,24 +1306,56 @@ namespace pxt { return cfg; } - filesToBePublishedAsync(allowPrivate = false) { + async filesToBePublishedAsync(allowPrivate = false, packExternalExtensions = false) { const files: Map = {}; - return this.loadAsync() - .then(() => { - if (!allowPrivate && !this.config.public) - U.userError('Only packages with "public":true can be published') - const cfg = this.prepareConfigToBePublished(); - files[pxt.CONFIG_NAME] = pxt.Package.stringifyConfig(cfg); - for (let f of this.getFiles()) { - // already stored - if (f == pxt.CONFIG_NAME) continue; - let str = this.readFile(f) - if (str == null) - U.userError("referenced file missing: " + f) - files[f] = str + await this.loadAsync(); + if (!allowPrivate && !this.config.public) + U.userError('Only packages with "public":true can be published') + const cfg = this.prepareConfigToBePublished(); + files[pxt.CONFIG_NAME] = pxt.Package.stringifyConfig(cfg); + + for (let f of this.getFiles()) { + // already stored + if (f == pxt.CONFIG_NAME) continue; + let str = this.readFile(f) + if (str == null) + U.userError("referenced file missing: " + f) + files[f] = str + } + + if (packExternalExtensions) { + const packDeps = (p: Package) => { + // package in current resolved version for use as backup + // e.g. loading hexfile back into editor when offline + const depsToPack = p.resolvedDependencies() + .filter(dep => { + switch (dep.verProtocol()) { + case "github": + case "pub": + return true; + default: + return false; + } + }); + for (const dep of depsToPack) { + // todo; swap this over to just one json blob under files, + // not keyed off -backup.json, to make extracting / hiding more obvious. key goes in pxtlib/main.ts + if (files[`${dep._verspec}-backup.json`]) + continue; + const packed: Map = {} + for (const toPack of dep.getFiles()) { + packed[toPack] = dep.readFile(toPack); + } + packed[pxt.CONFIG_NAME] = JSON.stringify(dep.config); + + files[`${dep._verspec}-backup.json`] = JSON.stringify(packed); + packDeps(dep); } - return U.sortObjectFields(files) - }) + } + + packDeps(this); + } + return U.sortObjectFields(files); } saveToJsonAsync(): Promise { From 404d63ad0ef4289cd559299d4d3b3685ef396ec1 Mon Sep 17 00:00:00 2001 From: Joey Wunderlich Date: Tue, 23 May 2023 10:33:03 -0700 Subject: [PATCH 2/9] Committing checkpoint to revert; this approach is messy, outlined better potential approach in pxtlib/package.ts --- pxtcompiler/emitter/driver.ts | 1 + pxtcompiler/emitter/hexfile.ts | 1 + pxtlib/github.ts | 14 ++-- pxtlib/main.ts | 2 +- pxtlib/package.ts | 120 ++++++++++++++++++++++++++------- webapp/src/app.tsx | 2 +- webapp/src/package.ts | 22 +++++- webapp/src/workspace.ts | 3 +- 8 files changed, 130 insertions(+), 35 deletions(-) diff --git a/pxtcompiler/emitter/driver.ts b/pxtcompiler/emitter/driver.ts index b9c11e4e5f13..de3fdf2e92a0 100644 --- a/pxtcompiler/emitter/driver.ts +++ b/pxtcompiler/emitter/driver.ts @@ -271,6 +271,7 @@ namespace ts.pxtc { if (opts.ast || opts.forceEmit || res.diagnostics.length == 0) { const binOutput = compileBinary(program, opts, res, entryPoint); + res.times["compilebinary"] = U.cpuUs() - emitStart res.diagnostics = res.diagnostics.concat(patchUpDiagnostics(binOutput.diagnostics)) } diff --git a/pxtcompiler/emitter/hexfile.ts b/pxtcompiler/emitter/hexfile.ts index 9be9a76df009..bef33dc7b891 100644 --- a/pxtcompiler/emitter/hexfile.ts +++ b/pxtcompiler/emitter/hexfile.ts @@ -1083,6 +1083,7 @@ _stored_program: .hex ${res} src = asmHeader(bin) + src } if (opts.embedBlob) { + bin.packedSource = packSource(opts.embedMeta, ts.pxtc.decodeBase64(opts.embedBlob)) // TODO more dynamic check for source size if (!bin.target.noSourceInFlash && bin.packedSource.length < 40000) { diff --git a/pxtlib/github.ts b/pxtlib/github.ts index 600014b792b4..f7addc208f35 100644 --- a/pxtlib/github.ts +++ b/pxtlib/github.ts @@ -630,7 +630,7 @@ namespace pxt.github { return { version, config }; } - export async function cacheProjectDependenciesAsync(cfg: pxt.PackageConfig): Promise { + export async function cacheProjectDependenciesAsync(cfg: pxt.PackageConfig, packagedFiles?: pxt.Map): Promise { const ghExtensions = Object.keys(cfg.dependencies) ?.filter(dep => isGithubId(cfg.dependencies[dep])); @@ -641,9 +641,15 @@ namespace pxt.github { ghExtensions.map( async ext => { const extSrc = cfg.dependencies[ext]; - const ghPkg = await downloadPackageAsync(extSrc, pkgConfig); - if (!ghPkg) { - throw new Error(lf("Cannot load extension {0} from {1}", ext, extSrc)); + let ghPkg: CachedPackage; + let caughtError: any; + try { + ghPkg = await downloadPackageAsync(extSrc, pkgConfig); + } catch (e) { + caughtError = e; + } + if ((!ghPkg || caughtError) && !packagedFiles?.[`${extSrc}-backup.json`]) { + throw caughtError || new Error(lf("Cannot load extension {0} from {1}", ext, extSrc)); } } ) diff --git a/pxtlib/main.ts b/pxtlib/main.ts index 81451cc05338..8e869d1f17a3 100644 --- a/pxtlib/main.ts +++ b/pxtlib/main.ts @@ -439,7 +439,7 @@ namespace pxt { export interface Host { readFile(pkg: Package, filename: string, skipAdditionalFiles?: boolean): string; writeFile(pkg: Package, filename: string, contents: string, force?: boolean): void; - downloadPackageAsync(pkg: Package, deps?: string[]): Promise; + downloadPackageAsync(pkg: Package, deps?: string[]/**, fallback?: () => void **/): Promise; getHexInfoAsync(extInfo: pxtc.ExtensionInfo): Promise; cacheStoreAsync(id: string, val: string): Promise; cacheGetAsync(id: string): Promise; // null if not found diff --git a/pxtlib/package.ts b/pxtlib/package.ts index 9ca53e06ccc1..4f7521cea791 100644 --- a/pxtlib/package.ts +++ b/pxtlib/package.ts @@ -142,12 +142,35 @@ namespace pxt { const proto = this.verProtocol() let files: Map; - if (proto == "pub") { - files = await Cloud.downloadScriptFilesAsync(this.verArgument()) - } else if (proto == "github") { - const config = await pxt.packagesConfigAsync(); - const resp = await pxt.github.downloadPackageAsync(this.verArgument(), config) - files = resp.files; + if (proto == "pub" || proto == "github") { + let caughtError: any; + try { + if (proto == "pub") { + files = await Cloud.downloadScriptFilesAsync(this.verArgument()) + } else { + const config = await pxt.packagesConfigAsync(); + const resp = await pxt.github.downloadPackageAsync(this.verArgument(), config) + files = resp.files; + } + } catch (e) { + caughtError = e ; + } + + // attempt to grab from top level package if `${dep._verspec}-backup.json` defined + let p = this.parent; + while (p && p != this) { + const backupFile = p.readFile(`${this._verspec}-backup.json`); + if (backupFile) { + files = JSON.parse(backupFile); + break; + } + p = p.parent; + } + + if (!files && caughtError) { + // no backup found, rethrow + throw caughtError || new Error("Could not download."); + } } else if (proto == "embed") { files = pxt.getEmbeddedScript(this.verArgument()) } else if (proto == "pkg") { @@ -307,27 +330,70 @@ namespace pxt { return Promise.resolve(v) } - private downloadAsync() { - return this.resolveVersionAsync() - .then(verNo => { - if (this.invalid()) { - pxt.debug(`skip download of invalid package ${this.id}`); - return undefined; - } - if (!/^embed:/.test(verNo) && this.installedVersion == verNo) - return undefined; - pxt.debug('downloading ' + verNo) - return this.host().downloadPackageAsync(this) - .then(() => { - this.loadConfig(); - - if (this.isAssetPack()) { - this.writeAssetPackFiles(); - } - pxt.debug(`installed ${this.id} /${verNo}`) - }) + /** + * strip out at beginning of hex file import + * push them into hex, github cache if those values are not already there + * flag attached that says they're temporary / backup only + * treat the cache as expired by time + * keep trying to fetch new one but return it in place until network is available + **/ + private async downloadAsync(): Promise { + const verNo = await this.resolveVersionAsync(); + if (this.invalid()) { + pxt.debug(`skip download of invalid package ${this.id}`); + return undefined; + } + if (!/^embed:/.test(verNo) && this.installedVersion == verNo) + return undefined; + pxt.debug('downloading ' + verNo) + await this.host().downloadPackageAsync( + this, + // undefined, + // () => { + // let p = this.parent; + // while (p && p != this) { + // const backupFile = p.readFile(`${this._verspec}-backup.json`); + // if (backupFile) { + // return JSON.parse(backupFile); + // } + // p = p.parent; + // } + + // return undefined; + // } + ); + // try { + // } catch (e) { + // let files: Map; + // let p = this.parent; + // while (p && p != this) { + // const backupFile = p.readFile(`${this._verspec}-backup.json`); + // if (backupFile) { + // files = JSON.parse(backupFile); + // break; + // } + // p = p.parent; + // } + + // if (files) { + // // this.writeFile(pxt.CONFIG_NAME, files[pxt.CONFIG_NAME]) + // // for (const fname of Object.keys(files)) { + // // this.writeFile(fname, files[fname]); + // // } + // // this + // } else { + // // no backup found, rethrow + // throw e; + // } + // } + + this.loadConfig(); + + if (this.isAssetPack()) { + this.writeAssetPackFiles(); + } - }) + pxt.debug(`installed ${this.id} /${verNo}`) } loadConfig() { @@ -1340,6 +1406,8 @@ namespace pxt { for (const dep of depsToPack) { // todo; swap this over to just one json blob under files, // not keyed off -backup.json, to make extracting / hiding more obvious. key goes in pxtlib/main.ts + + // include c++ pxtc.HexInfo as well? if (files[`${dep._verspec}-backup.json`]) continue; const packed: Map = {} diff --git a/webapp/src/app.tsx b/webapp/src/app.tsx index 6e355c0b3eb8..ce28ff565aad 100644 --- a/webapp/src/app.tsx +++ b/webapp/src/app.tsx @@ -2868,7 +2868,7 @@ export class ProjectView } files[pxt.CONFIG_NAME] = pxt.Package.stringifyConfig(cfg); - await pxt.github.cacheProjectDependenciesAsync(cfg); + await pxt.github.cacheProjectDependenciesAsync(cfg, files); const hd = await workspace.installAsync({ name: cfg.name, diff --git a/webapp/src/package.ts b/webapp/src/package.ts index b4a83de08f0d..1e6c5a4b760f 100644 --- a/webapp/src/package.ts +++ b/webapp/src/package.ts @@ -456,7 +456,13 @@ export class EditorPackage { sortedFiles(): File[] { let lst = Util.values(this.files) if (!pxt.options.debug) - lst = lst.filter(f => f.name != pxt.github.GIT_JSON && f.name != pxt.SIMSTATE_JSON && f.name != pxt.SERIAL_EDITOR_FILE && f.name != pxt.PALETTES_FILE) + lst = lst.filter(f => + f.name != pxt.github.GIT_JSON + && f.name != pxt.SIMSTATE_JSON + && f.name != pxt.SERIAL_EDITOR_FILE + && f.name != pxt.PALETTES_FILE + && !/(pub|github):.+-backup.json/.test(f.name) + ); lst.sort((a, b) => a.weight() - b.weight() || Util.strcmp(a.name, b.name)) return lst } @@ -701,7 +707,7 @@ class Host .then(v => v.val, e => null) } - downloadPackageAsync(pkg: pxt.Package): Promise { + downloadPackageAsync(pkg: pxt.Package, deps?: string[]/**, fallback?: () => void**/): Promise { let proto = pkg.verProtocol() let epkg = getEditorPkg(pkg) @@ -723,13 +729,25 @@ class Host } }) + const handleMissingNetwork = (e: any) => { + // const files = fallback(); + const files = mainPkg.readFile(`${pkg._verspec}-backup.json`); + if (files) { + epkg.setFiles(JSON.parse(files)); + } else { + throw e; + } + } + if (proto == "pub") { // make sure it sits in cache return workspace.getPublishedScriptAsync(pkg.verArgument()) .then(files => epkg.setFiles(files)) + .catch(handleMissingNetwork); } else if (proto == "github") { return workspace.getPublishedScriptAsync(pkg.version()) .then(files => epkg.setFiles(files)) + .catch(handleMissingNetwork); } else if (proto == "workspace") { return fromWorkspaceAsync(pkg.verArgument()) } else if (proto == "file") { diff --git a/webapp/src/workspace.ts b/webapp/src/workspace.ts index bfbde0a7a5fb..7c42409172cb 100644 --- a/webapp/src/workspace.ts +++ b/webapp/src/workspace.ts @@ -646,7 +646,8 @@ export function installAsync(h0: InstallHeader, text: ScriptText, dontOverwriteI pxt.shell.setEditorLanguagePref(cfg.preferredEditor); } - return pxt.github.cacheProjectDependenciesAsync(cfg) + + return pxt.github.cacheProjectDependenciesAsync(cfg, text) .then(() => importAsync(h, text)) .then(() => h); } From 75b7d178e45a8c1e96ba6c113617b00d893fff8e Mon Sep 17 00:00:00 2001 From: Joey Wunderlich Date: Tue, 23 May 2023 10:33:36 -0700 Subject: [PATCH 3/9] Revert "Committing checkpoint to revert; this approach is messy, outlined better potential approach in pxtlib/package.ts" This reverts commit 404d63ad0ef4289cd559299d4d3b3685ef396ec1. --- pxtcompiler/emitter/driver.ts | 1 - pxtcompiler/emitter/hexfile.ts | 1 - pxtlib/github.ts | 14 ++-- pxtlib/main.ts | 2 +- pxtlib/package.ts | 120 +++++++-------------------------- webapp/src/app.tsx | 2 +- webapp/src/package.ts | 22 +----- webapp/src/workspace.ts | 3 +- 8 files changed, 35 insertions(+), 130 deletions(-) diff --git a/pxtcompiler/emitter/driver.ts b/pxtcompiler/emitter/driver.ts index de3fdf2e92a0..b9c11e4e5f13 100644 --- a/pxtcompiler/emitter/driver.ts +++ b/pxtcompiler/emitter/driver.ts @@ -271,7 +271,6 @@ namespace ts.pxtc { if (opts.ast || opts.forceEmit || res.diagnostics.length == 0) { const binOutput = compileBinary(program, opts, res, entryPoint); - res.times["compilebinary"] = U.cpuUs() - emitStart res.diagnostics = res.diagnostics.concat(patchUpDiagnostics(binOutput.diagnostics)) } diff --git a/pxtcompiler/emitter/hexfile.ts b/pxtcompiler/emitter/hexfile.ts index bef33dc7b891..9be9a76df009 100644 --- a/pxtcompiler/emitter/hexfile.ts +++ b/pxtcompiler/emitter/hexfile.ts @@ -1083,7 +1083,6 @@ _stored_program: .hex ${res} src = asmHeader(bin) + src } if (opts.embedBlob) { - bin.packedSource = packSource(opts.embedMeta, ts.pxtc.decodeBase64(opts.embedBlob)) // TODO more dynamic check for source size if (!bin.target.noSourceInFlash && bin.packedSource.length < 40000) { diff --git a/pxtlib/github.ts b/pxtlib/github.ts index f7addc208f35..600014b792b4 100644 --- a/pxtlib/github.ts +++ b/pxtlib/github.ts @@ -630,7 +630,7 @@ namespace pxt.github { return { version, config }; } - export async function cacheProjectDependenciesAsync(cfg: pxt.PackageConfig, packagedFiles?: pxt.Map): Promise { + export async function cacheProjectDependenciesAsync(cfg: pxt.PackageConfig): Promise { const ghExtensions = Object.keys(cfg.dependencies) ?.filter(dep => isGithubId(cfg.dependencies[dep])); @@ -641,15 +641,9 @@ namespace pxt.github { ghExtensions.map( async ext => { const extSrc = cfg.dependencies[ext]; - let ghPkg: CachedPackage; - let caughtError: any; - try { - ghPkg = await downloadPackageAsync(extSrc, pkgConfig); - } catch (e) { - caughtError = e; - } - if ((!ghPkg || caughtError) && !packagedFiles?.[`${extSrc}-backup.json`]) { - throw caughtError || new Error(lf("Cannot load extension {0} from {1}", ext, extSrc)); + const ghPkg = await downloadPackageAsync(extSrc, pkgConfig); + if (!ghPkg) { + throw new Error(lf("Cannot load extension {0} from {1}", ext, extSrc)); } } ) diff --git a/pxtlib/main.ts b/pxtlib/main.ts index 8e869d1f17a3..81451cc05338 100644 --- a/pxtlib/main.ts +++ b/pxtlib/main.ts @@ -439,7 +439,7 @@ namespace pxt { export interface Host { readFile(pkg: Package, filename: string, skipAdditionalFiles?: boolean): string; writeFile(pkg: Package, filename: string, contents: string, force?: boolean): void; - downloadPackageAsync(pkg: Package, deps?: string[]/**, fallback?: () => void **/): Promise; + downloadPackageAsync(pkg: Package, deps?: string[]): Promise; getHexInfoAsync(extInfo: pxtc.ExtensionInfo): Promise; cacheStoreAsync(id: string, val: string): Promise; cacheGetAsync(id: string): Promise; // null if not found diff --git a/pxtlib/package.ts b/pxtlib/package.ts index 4f7521cea791..9ca53e06ccc1 100644 --- a/pxtlib/package.ts +++ b/pxtlib/package.ts @@ -142,35 +142,12 @@ namespace pxt { const proto = this.verProtocol() let files: Map; - if (proto == "pub" || proto == "github") { - let caughtError: any; - try { - if (proto == "pub") { - files = await Cloud.downloadScriptFilesAsync(this.verArgument()) - } else { - const config = await pxt.packagesConfigAsync(); - const resp = await pxt.github.downloadPackageAsync(this.verArgument(), config) - files = resp.files; - } - } catch (e) { - caughtError = e ; - } - - // attempt to grab from top level package if `${dep._verspec}-backup.json` defined - let p = this.parent; - while (p && p != this) { - const backupFile = p.readFile(`${this._verspec}-backup.json`); - if (backupFile) { - files = JSON.parse(backupFile); - break; - } - p = p.parent; - } - - if (!files && caughtError) { - // no backup found, rethrow - throw caughtError || new Error("Could not download."); - } + if (proto == "pub") { + files = await Cloud.downloadScriptFilesAsync(this.verArgument()) + } else if (proto == "github") { + const config = await pxt.packagesConfigAsync(); + const resp = await pxt.github.downloadPackageAsync(this.verArgument(), config) + files = resp.files; } else if (proto == "embed") { files = pxt.getEmbeddedScript(this.verArgument()) } else if (proto == "pkg") { @@ -330,70 +307,27 @@ namespace pxt { return Promise.resolve(v) } - /** - * strip out at beginning of hex file import - * push them into hex, github cache if those values are not already there - * flag attached that says they're temporary / backup only - * treat the cache as expired by time - * keep trying to fetch new one but return it in place until network is available - **/ - private async downloadAsync(): Promise { - const verNo = await this.resolveVersionAsync(); - if (this.invalid()) { - pxt.debug(`skip download of invalid package ${this.id}`); - return undefined; - } - if (!/^embed:/.test(verNo) && this.installedVersion == verNo) - return undefined; - pxt.debug('downloading ' + verNo) - await this.host().downloadPackageAsync( - this, - // undefined, - // () => { - // let p = this.parent; - // while (p && p != this) { - // const backupFile = p.readFile(`${this._verspec}-backup.json`); - // if (backupFile) { - // return JSON.parse(backupFile); - // } - // p = p.parent; - // } - - // return undefined; - // } - ); - // try { - // } catch (e) { - // let files: Map; - // let p = this.parent; - // while (p && p != this) { - // const backupFile = p.readFile(`${this._verspec}-backup.json`); - // if (backupFile) { - // files = JSON.parse(backupFile); - // break; - // } - // p = p.parent; - // } - - // if (files) { - // // this.writeFile(pxt.CONFIG_NAME, files[pxt.CONFIG_NAME]) - // // for (const fname of Object.keys(files)) { - // // this.writeFile(fname, files[fname]); - // // } - // // this - // } else { - // // no backup found, rethrow - // throw e; - // } - // } - - this.loadConfig(); - - if (this.isAssetPack()) { - this.writeAssetPackFiles(); - } + private downloadAsync() { + return this.resolveVersionAsync() + .then(verNo => { + if (this.invalid()) { + pxt.debug(`skip download of invalid package ${this.id}`); + return undefined; + } + if (!/^embed:/.test(verNo) && this.installedVersion == verNo) + return undefined; + pxt.debug('downloading ' + verNo) + return this.host().downloadPackageAsync(this) + .then(() => { + this.loadConfig(); + + if (this.isAssetPack()) { + this.writeAssetPackFiles(); + } + pxt.debug(`installed ${this.id} /${verNo}`) + }) - pxt.debug(`installed ${this.id} /${verNo}`) + }) } loadConfig() { @@ -1406,8 +1340,6 @@ namespace pxt { for (const dep of depsToPack) { // todo; swap this over to just one json blob under files, // not keyed off -backup.json, to make extracting / hiding more obvious. key goes in pxtlib/main.ts - - // include c++ pxtc.HexInfo as well? if (files[`${dep._verspec}-backup.json`]) continue; const packed: Map = {} diff --git a/webapp/src/app.tsx b/webapp/src/app.tsx index ce28ff565aad..6e355c0b3eb8 100644 --- a/webapp/src/app.tsx +++ b/webapp/src/app.tsx @@ -2868,7 +2868,7 @@ export class ProjectView } files[pxt.CONFIG_NAME] = pxt.Package.stringifyConfig(cfg); - await pxt.github.cacheProjectDependenciesAsync(cfg, files); + await pxt.github.cacheProjectDependenciesAsync(cfg); const hd = await workspace.installAsync({ name: cfg.name, diff --git a/webapp/src/package.ts b/webapp/src/package.ts index 1e6c5a4b760f..b4a83de08f0d 100644 --- a/webapp/src/package.ts +++ b/webapp/src/package.ts @@ -456,13 +456,7 @@ export class EditorPackage { sortedFiles(): File[] { let lst = Util.values(this.files) if (!pxt.options.debug) - lst = lst.filter(f => - f.name != pxt.github.GIT_JSON - && f.name != pxt.SIMSTATE_JSON - && f.name != pxt.SERIAL_EDITOR_FILE - && f.name != pxt.PALETTES_FILE - && !/(pub|github):.+-backup.json/.test(f.name) - ); + lst = lst.filter(f => f.name != pxt.github.GIT_JSON && f.name != pxt.SIMSTATE_JSON && f.name != pxt.SERIAL_EDITOR_FILE && f.name != pxt.PALETTES_FILE) lst.sort((a, b) => a.weight() - b.weight() || Util.strcmp(a.name, b.name)) return lst } @@ -707,7 +701,7 @@ class Host .then(v => v.val, e => null) } - downloadPackageAsync(pkg: pxt.Package, deps?: string[]/**, fallback?: () => void**/): Promise { + downloadPackageAsync(pkg: pxt.Package): Promise { let proto = pkg.verProtocol() let epkg = getEditorPkg(pkg) @@ -729,25 +723,13 @@ class Host } }) - const handleMissingNetwork = (e: any) => { - // const files = fallback(); - const files = mainPkg.readFile(`${pkg._verspec}-backup.json`); - if (files) { - epkg.setFiles(JSON.parse(files)); - } else { - throw e; - } - } - if (proto == "pub") { // make sure it sits in cache return workspace.getPublishedScriptAsync(pkg.verArgument()) .then(files => epkg.setFiles(files)) - .catch(handleMissingNetwork); } else if (proto == "github") { return workspace.getPublishedScriptAsync(pkg.version()) .then(files => epkg.setFiles(files)) - .catch(handleMissingNetwork); } else if (proto == "workspace") { return fromWorkspaceAsync(pkg.verArgument()) } else if (proto == "file") { diff --git a/webapp/src/workspace.ts b/webapp/src/workspace.ts index 7c42409172cb..bfbde0a7a5fb 100644 --- a/webapp/src/workspace.ts +++ b/webapp/src/workspace.ts @@ -646,8 +646,7 @@ export function installAsync(h0: InstallHeader, text: ScriptText, dontOverwriteI pxt.shell.setEditorLanguagePref(cfg.preferredEditor); } - - return pxt.github.cacheProjectDependenciesAsync(cfg, text) + return pxt.github.cacheProjectDependenciesAsync(cfg) .then(() => importAsync(h, text)) .then(() => h); } From e0d47a0edd74bbd12627786a1693f29ec1abc839 Mon Sep 17 00:00:00 2001 From: Joey Wunderlich Date: Tue, 23 May 2023 19:20:41 -0700 Subject: [PATCH 4/9] pass backup script text thru to github store, mark, pack into 1 subfile --- pxtlib/github.ts | 97 +++++++++++++++++++++++------------------ pxtlib/main.ts | 2 + pxtlib/package.ts | 25 ++++++++--- webapp/src/db.ts | 42 +++++++++--------- webapp/src/workspace.ts | 16 ++++++- 5 files changed, 113 insertions(+), 69 deletions(-) diff --git a/pxtlib/github.ts b/pxtlib/github.ts index 600014b792b4..3de86b507ca4 100644 --- a/pxtlib/github.ts +++ b/pxtlib/github.ts @@ -101,13 +101,14 @@ namespace pxt.github { export interface CachedPackage { files: Map; + backupCopy?: boolean; } // caching export interface IGithubDb { latestVersionAsync(repopath: string, config: PackagesConfig): Promise; loadConfigAsync(repopath: string, tag: string): Promise; - loadPackageAsync(repopath: string, tag: string): Promise; + loadPackageAsync(repopath: string, tag: string, backupScriptText?: pxt.Map): Promise; } function ghRequestAsync(options: U.HttpRequestOptions) { @@ -168,19 +169,19 @@ namespace pxt.github { private configs: pxt.Map = {}; private packages: pxt.Map = {}; - private proxyWithCdnLoadPackageAsync(repopath: string, tag: string): Promise { + private async proxyWithCdnLoadPackageAsync(repopath: string, tag: string): Promise { // cache lookup const key = `${repopath}/${tag}`; let res = this.packages[key]; if (res) { pxt.debug(`github cache ${repopath}/${tag}/text`); - return Promise.resolve(res); + return res; } // load and cache - const parsed = parseRepoId(repopath) - return ghProxyWithCdnJsonAsync(join(parsed.slug, tag, parsed.fileName, "text")) - .then(v => this.packages[key] = { files: v }); + const parsed = parseRepoId(repopath); + const v = await ghProxyWithCdnJsonAsync(join(parsed.slug, tag, parsed.fileName, "text")); + return this.packages[key] = { files: v }; } private cacheConfig(key: string, v: string) { @@ -227,7 +228,7 @@ namespace pxt.github { return resolved } - async loadPackageAsync(repopath: string, tag: string): Promise { + async loadPackageAsync(repopath: string, tag: string, backupScriptText?: pxt.Map): Promise { if (!tag) { pxt.debug(`load pkg: default to master branch`) tag = "master"; @@ -243,41 +244,48 @@ namespace pxt.github { } // try using github apis - return await this.githubLoadPackageAsync(repopath, tag); + return await this.githubLoadPackageAsync(repopath, tag, backupScriptText); } - private githubLoadPackageAsync(repopath: string, tag: string): Promise { - return tagToShaAsync(repopath, tag) - .then(sha => { - // cache lookup - const key = `${repopath}/${sha}`; - let res = this.packages[key]; - if (res) { - pxt.debug(`github cache ${repopath}/${tag}/text`); - return Promise.resolve(U.clone(res)); + private async githubLoadPackageAsync(repopath: string, tag: string, backupScriptText?: pxt.Map): Promise { + // load and cache + const current: CachedPackage = { + files: {} + } + const key = `${repopath}/${tag}`; + // ^^ double check this diff; is there a reason to store keyed off sha? just for master special case maybe? + try { + const sha = await tagToShaAsync(repopath, tag); + // cache lookup + let res = this.packages[key]; + if (res) { + pxt.debug(`github cache ${repopath}/${tag}/text`); + return U.clone(res); + } + + pxt.log(`Downloading ${repopath}/${tag} -> ${sha}`); + const pkg = await downloadTextAsync(repopath, sha, pxt.CONFIG_NAME); + current.files[pxt.CONFIG_NAME] = pkg + const cfg: pxt.PackageConfig = JSON.parse(pkg) + await U.promiseMapAll( + pxt.allPkgFiles(cfg).slice(1), + async fn => { + const text = await downloadTextAsync(repopath, sha, fn); + current.files[fn] = text; } + ) + } catch (e) { + if (backupScriptText) { + current.files = U.clone(backupScriptText); + current.backupCopy = true; + } else { + throw e; + } + } - // load and cache - pxt.log(`Downloading ${repopath}/${tag} -> ${sha}`) - return downloadTextAsync(repopath, sha, pxt.CONFIG_NAME) - .then(pkg => { - const current: CachedPackage = { - files: {} - } - current.files[pxt.CONFIG_NAME] = pkg - const cfg: pxt.PackageConfig = JSON.parse(pkg) - return U.promiseMapAll(pxt.allPkgFiles(cfg).slice(1), - fn => downloadTextAsync(repopath, sha, fn) - .then(text => { - current.files[fn] = text - })) - .then(() => { - // cache! - this.packages[key] = current; - return U.clone(current); - }) - }) - }) + // cache! + this.packages[key] = current; + return U.clone(current); } } @@ -586,7 +594,7 @@ namespace pxt.github { return await db.loadConfigAsync(repopath, tag) } - export async function downloadPackageAsync(repoWithTag: string, config: pxt.PackagesConfig): Promise { + export async function downloadPackageAsync(repoWithTag: string, config: pxt.PackagesConfig, backupScriptText?: pxt.Map): Promise { const p = parseRepoId(repoWithTag) if (!p) { pxt.log('Unknown GitHub syntax'); @@ -599,12 +607,13 @@ namespace pxt.github { return undefined; } + // TODO check if this needs adjustments / special casing for backupScriptText; try catch around it? // always try to upgrade unbound versions if (!p.tag) { p.tag = await db.latestVersionAsync(p.slug, config) } - const cached = await db.loadPackageAsync(p.fullName, p.tag) - const dv = upgradedDisablesVariants(config, repoWithTag) + const cached = await db.loadPackageAsync(p.fullName, p.tag, backupScriptText); + const dv = upgradedDisablesVariants(config, repoWithTag); if (dv) { const cfg = Package.parseAndValidConfig(cached.files[pxt.CONFIG_NAME]) if (cfg) { @@ -630,10 +639,11 @@ namespace pxt.github { return { version, config }; } - export async function cacheProjectDependenciesAsync(cfg: pxt.PackageConfig): Promise { + export async function cacheProjectDependenciesAsync(cfg: pxt.PackageConfig, backupExtensions?: pxt.Map>): Promise { const ghExtensions = Object.keys(cfg.dependencies) ?.filter(dep => isGithubId(cfg.dependencies[dep])); + // todo: handle nested deps; recurse? if (ghExtensions.length) { const pkgConfig = await pxt.packagesConfigAsync(); // Make sure external packages load before installing header. @@ -641,7 +651,8 @@ namespace pxt.github { ghExtensions.map( async ext => { const extSrc = cfg.dependencies[ext]; - const ghPkg = await downloadPackageAsync(extSrc, pkgConfig); + const backup = backupExtensions?.[extSrc]; + const ghPkg = await downloadPackageAsync(extSrc, pkgConfig, backup); if (!ghPkg) { throw new Error(lf("Cannot load extension {0} from {1}", ext, extSrc)); } diff --git a/pxtlib/main.ts b/pxtlib/main.ts index 81451cc05338..44a88c8f9c5c 100644 --- a/pxtlib/main.ts +++ b/pxtlib/main.ts @@ -512,6 +512,8 @@ namespace pxt { export const TUTORIAL_CUSTOM_TS = "tutorial.custom.ts"; export const BREAKPOINT_TABLET = 991; // TODO (shakao) revisit when tutorial stuff is more settled export const PALETTES_FILE = "_palettes.json"; + // for packing extensions into distributables, as backup when network unavailable + export const PACKAGED_EXTENSIONS = "_packaged-extensions.json"; export function outputName(trg: pxtc.CompileTarget = null) { if (!trg) trg = appTarget.compile diff --git a/pxtlib/package.ts b/pxtlib/package.ts index 9ca53e06ccc1..93ad144a6b49 100644 --- a/pxtlib/package.ts +++ b/pxtlib/package.ts @@ -1324,6 +1324,16 @@ namespace pxt { } if (packExternalExtensions) { + let filesToEmit = false; + // todo throw this in a common spot + const packedDeps: { + extensionText?: Map>, + hex?: Map, + } = { + extensionText: {}, + hex: {}, + } + const packDeps = (p: Package) => { // package in current resolved version for use as backup // e.g. loading hexfile back into editor when offline @@ -1337,23 +1347,28 @@ namespace pxt { return false; } }); + for (const dep of depsToPack) { - // todo; swap this over to just one json blob under files, - // not keyed off -backup.json, to make extracting / hiding more obvious. key goes in pxtlib/main.ts - if (files[`${dep._verspec}-backup.json`]) + if (packedDeps.extensionText[dep._verspec]) continue; - const packed: Map = {} + const packed: Map = {}; for (const toPack of dep.getFiles()) { packed[toPack] = dep.readFile(toPack); } packed[pxt.CONFIG_NAME] = JSON.stringify(dep.config); - files[`${dep._verspec}-backup.json`] = JSON.stringify(packed); + packedDeps.extensionText[dep._verspec] = packed; + filesToEmit = true; packDeps(dep); } } packDeps(this); + if (filesToEmit) { + if (!Object.keys(packedDeps.hex).length) + delete packedDeps["hex"]; + files[pxt.PACKAGED_EXTENSIONS] = JSON.stringify(packedDeps); + } } return U.sortObjectFields(files); } diff --git a/webapp/src/db.ts b/webapp/src/db.ts index 4260ff18d985..aab3d309a597 100644 --- a/webapp/src/db.ts +++ b/webapp/src/db.ts @@ -122,32 +122,34 @@ class GithubDb implements pxt.github.IGithubDb { } // not found ); } - loadPackageAsync(repopath: string, tag: string): Promise { + async loadPackageAsync(repopath: string, tag: string, backupScriptText?: pxt.Map): Promise { if (!tag) { - pxt.debug(`dep: default to master`) - tag = "master" + pxt.debug(`dep: default to master`) + tag = "master" } // don't cache master if (tag == "master") - return this.mem.loadPackageAsync(repopath, tag); + return this.mem.loadPackageAsync(repopath, tag, backupScriptText); const id = `pkg-${repopath}-${tag}`; - return this.table.getAsync(id).then( - entry => { - pxt.debug(`github offline cache hit ${id}`); - return entry.package as pxt.github.CachedPackage; - }, - e => { - pxt.debug(`github offline cache miss ${id}`); - return this.mem.loadPackageAsync(repopath, tag) - .then(p => { - return this.table.forceSetAsync({ - id, - package: p - }).then(() => p, e => p); - }) - } // not found - ); + try { + const entry = await this.table.getAsync(id); + pxt.debug(`github offline cache hit ${id}`); + // TODO: back up check here if .backupCopy to try fetch from this.mem? + return entry.package as pxt.github.CachedPackage; + + } catch (e) { + pxt.debug(`github offline cache miss ${id}`); + const p = await this.mem.loadPackageAsync(repopath, tag, backupScriptText); + try { + await this.table.forceSetAsync({ + id, + package: p + }); + } finally { + return p; + } + } } } diff --git a/webapp/src/workspace.ts b/webapp/src/workspace.ts index bfbde0a7a5fb..075326e64a74 100644 --- a/webapp/src/workspace.ts +++ b/webapp/src/workspace.ts @@ -646,7 +646,21 @@ export function installAsync(h0: InstallHeader, text: ScriptText, dontOverwriteI pxt.shell.setEditorLanguagePref(cfg.preferredEditor); } - return pxt.github.cacheProjectDependenciesAsync(cfg) + let backupExtensionFiles: pxt.Map>; + if (text[pxt.PACKAGED_EXTENSIONS]) { + const packagedExts: { + extensionText?: pxt.Map>, + hex?: pxt.Map, + } = pxt.Util.jsonTryParse(text[pxt.PACKAGED_EXTENSIONS]); + backupExtensionFiles = packagedExts?.extensionText; + const hexInfo = packagedExts?.hex; + // TODO + + // Do not persist into project once installed. + delete text[pxt.PACKAGED_EXTENSIONS]; + } + + return pxt.github.cacheProjectDependenciesAsync(cfg, backupExtensionFiles) .then(() => importAsync(h, text)) .then(() => h); } From 6fc7adada1e49af9c4ca21b4fb745488f75407df Mon Sep 17 00:00:00 2001 From: Joey Wunderlich Date: Tue, 23 May 2023 20:00:23 -0700 Subject: [PATCH 5/9] recursive check to handle nested gh deps --- pxtlib/github.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pxtlib/github.ts b/pxtlib/github.ts index 3de86b507ca4..1a06487834b2 100644 --- a/pxtlib/github.ts +++ b/pxtlib/github.ts @@ -639,7 +639,10 @@ namespace pxt.github { return { version, config }; } - export async function cacheProjectDependenciesAsync(cfg: pxt.PackageConfig, backupExtensions?: pxt.Map>): Promise { + export async function cacheProjectDependenciesAsync( + cfg: pxt.PackageConfig, + backupExtensions?: pxt.Map> + ): Promise { const ghExtensions = Object.keys(cfg.dependencies) ?.filter(dep => isGithubId(cfg.dependencies[dep])); @@ -656,6 +659,11 @@ namespace pxt.github { if (!ghPkg) { throw new Error(lf("Cannot load extension {0} from {1}", ext, extSrc)); } + + const pkgCfg = pxt.U.jsonTryParse(ghPkg.files[pxt.CONFIG_NAME]); + // todo: we don't support circular deps... right?? maybe add an inner function to + // recurse & keep track of already resolved deps + await cacheProjectDependenciesAsync(pkgCfg, backupExtensions); } ) ); From b906ef170725115e5061408389d19f2203a1baa7 Mon Sep 17 00:00:00 2001 From: Joey Wunderlich Date: Tue, 23 May 2023 20:42:11 -0700 Subject: [PATCH 6/9] don't return from finally since linter doesn't like it --- pxtlib/github.ts | 4 +++- pxtlib/package.ts | 2 +- webapp/src/db.ts | 5 +++-- webapp/src/workspace.ts | 1 + 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/pxtlib/github.ts b/pxtlib/github.ts index 1a06487834b2..5cecebf9b479 100644 --- a/pxtlib/github.ts +++ b/pxtlib/github.ts @@ -646,7 +646,9 @@ namespace pxt.github { const ghExtensions = Object.keys(cfg.dependencies) ?.filter(dep => isGithubId(cfg.dependencies[dep])); - // todo: handle nested deps; recurse? + // need to check/cache pub: links + inject to cache? + // probably fine to put off from initial pass as it's a bit of an edge case? + // maybe extend pxt.github.cache... a bit? if (ghExtensions.length) { const pkgConfig = await pxt.packagesConfigAsync(); // Make sure external packages load before installing header. diff --git a/pxtlib/package.ts b/pxtlib/package.ts index 93ad144a6b49..9691a06a68a0 100644 --- a/pxtlib/package.ts +++ b/pxtlib/package.ts @@ -1332,7 +1332,7 @@ namespace pxt { } = { extensionText: {}, hex: {}, - } + }; const packDeps = (p: Package) => { // package in current resolved version for use as backup diff --git a/webapp/src/db.ts b/webapp/src/db.ts index aab3d309a597..b5fbdb11db8c 100644 --- a/webapp/src/db.ts +++ b/webapp/src/db.ts @@ -146,9 +146,10 @@ class GithubDb implements pxt.github.IGithubDb { id, package: p }); - } finally { - return p; + } catch (e) { + // swallow caching error } + return p; } } } diff --git a/webapp/src/workspace.ts b/webapp/src/workspace.ts index 075326e64a74..eed0ea3ac2bb 100644 --- a/webapp/src/workspace.ts +++ b/webapp/src/workspace.ts @@ -652,6 +652,7 @@ export function installAsync(h0: InstallHeader, text: ScriptText, dontOverwriteI extensionText?: pxt.Map>, hex?: pxt.Map, } = pxt.Util.jsonTryParse(text[pxt.PACKAGED_EXTENSIONS]); + backupExtensionFiles = packagedExts?.extensionText; const hexInfo = packagedExts?.hex; // TODO From 48d5e139dbd8f95a8df63671ee001547aa165739 Mon Sep 17 00:00:00 2001 From: Joey Wunderlich Date: Tue, 13 Jun 2023 13:45:35 -0700 Subject: [PATCH 7/9] handle circ deps --- pxtlib/github.ts | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/pxtlib/github.ts b/pxtlib/github.ts index 5cecebf9b479..1ae54aa7f7a4 100644 --- a/pxtlib/github.ts +++ b/pxtlib/github.ts @@ -265,8 +265,8 @@ namespace pxt.github { pxt.log(`Downloading ${repopath}/${tag} -> ${sha}`); const pkg = await downloadTextAsync(repopath, sha, pxt.CONFIG_NAME); - current.files[pxt.CONFIG_NAME] = pkg - const cfg: pxt.PackageConfig = JSON.parse(pkg) + current.files[pxt.CONFIG_NAME] = pkg; + const cfg: pxt.PackageConfig = JSON.parse(pkg); await U.promiseMapAll( pxt.allPkgFiles(cfg).slice(1), async fn => { @@ -642,6 +642,18 @@ namespace pxt.github { export async function cacheProjectDependenciesAsync( cfg: pxt.PackageConfig, backupExtensions?: pxt.Map> + ): Promise { + return cacheProjectDependenciesAsyncCore( + cfg, + {} /** resolved */, + backupExtensions + ); + } + + async function cacheProjectDependenciesAsyncCore( + cfg: pxt.PackageConfig, + checked: pxt.Map, + backupExtensions?: pxt.Map> ): Promise { const ghExtensions = Object.keys(cfg.dependencies) ?.filter(dep => isGithubId(cfg.dependencies[dep])); @@ -656,6 +668,9 @@ namespace pxt.github { ghExtensions.map( async ext => { const extSrc = cfg.dependencies[ext]; + if (checked[extSrc]) + return; + checked[extSrc] = true; const backup = backupExtensions?.[extSrc]; const ghPkg = await downloadPackageAsync(extSrc, pkgConfig, backup); if (!ghPkg) { @@ -663,9 +678,7 @@ namespace pxt.github { } const pkgCfg = pxt.U.jsonTryParse(ghPkg.files[pxt.CONFIG_NAME]); - // todo: we don't support circular deps... right?? maybe add an inner function to - // recurse & keep track of already resolved deps - await cacheProjectDependenciesAsync(pkgCfg, backupExtensions); + await cacheProjectDependenciesAsyncCore(pkgCfg, checked, backupExtensions); } ) ); From 48d44caa3b9c6bab48c11bea66cbbb4e69f4a686 Mon Sep 17 00:00:00 2001 From: Joey Wunderlich Date: Tue, 13 Jun 2023 15:02:10 -0700 Subject: [PATCH 8/9] unwrap a bit so it's just deps, so hex info would be stored separately --- pxtlib/package.ts | 22 ++++------------------ webapp/src/workspace.ts | 12 ++++-------- 2 files changed, 8 insertions(+), 26 deletions(-) diff --git a/pxtlib/package.ts b/pxtlib/package.ts index 1ab5d77f133a..ca358d1ba05c 100644 --- a/pxtlib/package.ts +++ b/pxtlib/package.ts @@ -1336,19 +1336,8 @@ namespace pxt { } if (packExternalExtensions) { - let filesToEmit = false; - // todo throw this in a common spot - const packedDeps: { - extensionText?: Map>, - hex?: Map, - } = { - extensionText: {}, - hex: {}, - }; - + const packedDeps: Map> = {}; const packDeps = (p: Package) => { - // package in current resolved version for use as backup - // e.g. loading hexfile back into editor when offline const depsToPack = p.resolvedDependencies() .filter(dep => { switch (dep.verProtocol()) { @@ -1361,7 +1350,7 @@ namespace pxt { }); for (const dep of depsToPack) { - if (packedDeps.extensionText[dep._verspec]) + if (packedDeps[dep._verspec]) continue; const packed: Map = {}; for (const toPack of dep.getFiles()) { @@ -1369,16 +1358,13 @@ namespace pxt { } packed[pxt.CONFIG_NAME] = JSON.stringify(dep.config); - packedDeps.extensionText[dep._verspec] = packed; - filesToEmit = true; + packedDeps[dep._verspec] = packed; packDeps(dep); } } packDeps(this); - if (filesToEmit) { - if (!Object.keys(packedDeps.hex).length) - delete packedDeps["hex"]; + if (Object.keys(packedDeps).length) { files[pxt.PACKAGED_EXTENSIONS] = JSON.stringify(packedDeps); } } diff --git a/webapp/src/workspace.ts b/webapp/src/workspace.ts index eed0ea3ac2bb..0c89b8a84020 100644 --- a/webapp/src/workspace.ts +++ b/webapp/src/workspace.ts @@ -648,14 +648,10 @@ export function installAsync(h0: InstallHeader, text: ScriptText, dontOverwriteI let backupExtensionFiles: pxt.Map>; if (text[pxt.PACKAGED_EXTENSIONS]) { - const packagedExts: { - extensionText?: pxt.Map>, - hex?: pxt.Map, - } = pxt.Util.jsonTryParse(text[pxt.PACKAGED_EXTENSIONS]); - - backupExtensionFiles = packagedExts?.extensionText; - const hexInfo = packagedExts?.hex; - // TODO + backupExtensionFiles = pxt.Util.jsonTryParse(text[pxt.PACKAGED_EXTENSIONS]); + // TODO cache hexfiles in text[pxt.PACKAGED_HEXFILE] like field? + // Would be necessary for full offline usage, but tricky as it + // also depends on target version / hex hash; // Do not persist into project once installed. delete text[pxt.PACKAGED_EXTENSIONS]; From 5e71e5e9fd58825ee15c4eb9c2c8147b354b48f3 Mon Sep 17 00:00:00 2001 From: Joey Wunderlich Date: Tue, 13 Jun 2023 15:23:07 -0700 Subject: [PATCH 9/9] plug in portion mocking out where to read / write hex to / from --- pxtlib/main.ts | 1 + pxtlib/package.ts | 14 ++++++++++---- webapp/src/workspace.ts | 6 ++++++ 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/pxtlib/main.ts b/pxtlib/main.ts index 44a88c8f9c5c..21941c50a0c9 100644 --- a/pxtlib/main.ts +++ b/pxtlib/main.ts @@ -514,6 +514,7 @@ namespace pxt { export const PALETTES_FILE = "_palettes.json"; // for packing extensions into distributables, as backup when network unavailable export const PACKAGED_EXTENSIONS = "_packaged-extensions.json"; + export const PACKAGED_EXT_INFO = "_packaged-ext-info.json"; export function outputName(trg: pxtc.CompileTarget = null) { if (!trg) trg = appTarget.compile diff --git a/pxtlib/package.ts b/pxtlib/package.ts index ca358d1ba05c..72a29c27fd5a 100644 --- a/pxtlib/package.ts +++ b/pxtlib/package.ts @@ -1220,8 +1220,7 @@ namespace pxt { variants = [null] } - - let ext: pxtc.ExtensionInfo = null + let ext: pxtc.ExtensionInfo = null; for (let v of variants) { if (ext) pxt.debug(`building for ${v}`) @@ -1252,6 +1251,11 @@ namespace pxt { true, !appTarget.compile.useUF2 ); + if (opts.target.isNative && opts.extinfo.hexinfo) { + // todo trim down to relevant portion of extinfo? + // hexfile + hash + whatever is needed for /cpp.ts + files[pxt.PACKAGED_EXT_INFO] = JSON.stringify(opts.extinfo); + } const headerString = JSON.stringify({ name: this.config.name, comment: this.config.description, @@ -1261,7 +1265,9 @@ namespace pxt { editor: this.getPreferredEditor(), targetVersions: pxt.appTarget.versions }) - const programText = JSON.stringify(files) + + const programText = JSON.stringify(files); + const buf = await lzmaCompressAsync(headerString + programText) if (buf) { opts.embedMeta = JSON.stringify({ @@ -1272,7 +1278,7 @@ namespace pxt { eURL: pxt.appTarget.appTheme.embedUrl, eVER: pxt.appTarget.versions ? pxt.appTarget.versions.target : "", pxtTarget: appTarget.id, - }) + }); opts.embedBlob = ts.pxtc.encodeBase64(U.uint8ArrayToString(buf)) } } diff --git a/webapp/src/workspace.ts b/webapp/src/workspace.ts index 0c89b8a84020..6acdc5212040 100644 --- a/webapp/src/workspace.ts +++ b/webapp/src/workspace.ts @@ -656,6 +656,12 @@ export function installAsync(h0: InstallHeader, text: ScriptText, dontOverwriteI // Do not persist into project once installed. delete text[pxt.PACKAGED_EXTENSIONS]; } + if (text[pxt.PACKAGED_EXT_INFO]) { + const parsedExtensionInfo = pxt.Util.jsonTryParse(text[pxt.PACKAGED_EXT_INFO]); + // TODO: push this into cache; see pxtlib/cpp.ts, would need to mirror portion of + // getHexInfoAsync that compresses hex download result and pushes into storeWithLimitAsync + delete text[pxt.PACKAGED_EXT_INFO]; + } return pxt.github.cacheProjectDependenciesAsync(cfg, backupExtensionFiles) .then(() => importAsync(h, text))