diff --git a/pxtlib/github.ts b/pxtlib/github.ts index 600014b792b4..1ae54aa7f7a4 100644 --- a/pxtlib/github.ts +++ b/pxtlib/github.ts @@ -101,13 +101,14 @@ namespace pxt.github { export interface CachedPackage { files: Map; + backupCopy?: boolean; } // caching export interface IGithubDb { latestVersionAsync(repopath: string, config: PackagesConfig): Promise; loadConfigAsync(repopath: string, tag: string): Promise; - loadPackageAsync(repopath: string, tag: string): Promise; + loadPackageAsync(repopath: string, tag: string, backupScriptText?: pxt.Map): Promise; } function ghRequestAsync(options: U.HttpRequestOptions) { @@ -168,19 +169,19 @@ namespace pxt.github { private configs: pxt.Map = {}; private packages: pxt.Map = {}; - private proxyWithCdnLoadPackageAsync(repopath: string, tag: string): Promise { + private async proxyWithCdnLoadPackageAsync(repopath: string, tag: string): Promise { // cache lookup const key = `${repopath}/${tag}`; let res = this.packages[key]; if (res) { pxt.debug(`github cache ${repopath}/${tag}/text`); - return Promise.resolve(res); + return res; } // load and cache - const parsed = parseRepoId(repopath) - return ghProxyWithCdnJsonAsync(join(parsed.slug, tag, parsed.fileName, "text")) - .then(v => this.packages[key] = { files: v }); + const parsed = parseRepoId(repopath); + const v = await ghProxyWithCdnJsonAsync(join(parsed.slug, tag, parsed.fileName, "text")); + return this.packages[key] = { files: v }; } private cacheConfig(key: string, v: string) { @@ -227,7 +228,7 @@ namespace pxt.github { return resolved } - async loadPackageAsync(repopath: string, tag: string): Promise { + async loadPackageAsync(repopath: string, tag: string, backupScriptText?: pxt.Map): Promise { if (!tag) { pxt.debug(`load pkg: default to master branch`) tag = "master"; @@ -243,41 +244,48 @@ namespace pxt.github { } // try using github apis - return await this.githubLoadPackageAsync(repopath, tag); + return await this.githubLoadPackageAsync(repopath, tag, backupScriptText); } - private githubLoadPackageAsync(repopath: string, tag: string): Promise { - return tagToShaAsync(repopath, tag) - .then(sha => { - // cache lookup - const key = `${repopath}/${sha}`; - let res = this.packages[key]; - if (res) { - pxt.debug(`github cache ${repopath}/${tag}/text`); - return Promise.resolve(U.clone(res)); + private async githubLoadPackageAsync(repopath: string, tag: string, backupScriptText?: pxt.Map): Promise { + // load and cache + const current: CachedPackage = { + files: {} + } + const key = `${repopath}/${tag}`; + // ^^ double check this diff; is there a reason to store keyed off sha? just for master special case maybe? + try { + const sha = await tagToShaAsync(repopath, tag); + // cache lookup + let res = this.packages[key]; + if (res) { + pxt.debug(`github cache ${repopath}/${tag}/text`); + return U.clone(res); + } + + pxt.log(`Downloading ${repopath}/${tag} -> ${sha}`); + const pkg = await downloadTextAsync(repopath, sha, pxt.CONFIG_NAME); + current.files[pxt.CONFIG_NAME] = pkg; + const cfg: pxt.PackageConfig = JSON.parse(pkg); + await U.promiseMapAll( + pxt.allPkgFiles(cfg).slice(1), + async fn => { + const text = await downloadTextAsync(repopath, sha, fn); + current.files[fn] = text; } + ) + } catch (e) { + if (backupScriptText) { + current.files = U.clone(backupScriptText); + current.backupCopy = true; + } else { + throw e; + } + } - // load and cache - pxt.log(`Downloading ${repopath}/${tag} -> ${sha}`) - return downloadTextAsync(repopath, sha, pxt.CONFIG_NAME) - .then(pkg => { - const current: CachedPackage = { - files: {} - } - current.files[pxt.CONFIG_NAME] = pkg - const cfg: pxt.PackageConfig = JSON.parse(pkg) - return U.promiseMapAll(pxt.allPkgFiles(cfg).slice(1), - fn => downloadTextAsync(repopath, sha, fn) - .then(text => { - current.files[fn] = text - })) - .then(() => { - // cache! - this.packages[key] = current; - return U.clone(current); - }) - }) - }) + // cache! + this.packages[key] = current; + return U.clone(current); } } @@ -586,7 +594,7 @@ namespace pxt.github { return await db.loadConfigAsync(repopath, tag) } - export async function downloadPackageAsync(repoWithTag: string, config: pxt.PackagesConfig): Promise { + export async function downloadPackageAsync(repoWithTag: string, config: pxt.PackagesConfig, backupScriptText?: pxt.Map): Promise { const p = parseRepoId(repoWithTag) if (!p) { pxt.log('Unknown GitHub syntax'); @@ -599,12 +607,13 @@ namespace pxt.github { return undefined; } + // TODO check if this needs adjustments / special casing for backupScriptText; try catch around it? // always try to upgrade unbound versions if (!p.tag) { p.tag = await db.latestVersionAsync(p.slug, config) } - const cached = await db.loadPackageAsync(p.fullName, p.tag) - const dv = upgradedDisablesVariants(config, repoWithTag) + const cached = await db.loadPackageAsync(p.fullName, p.tag, backupScriptText); + const dv = upgradedDisablesVariants(config, repoWithTag); if (dv) { const cfg = Package.parseAndValidConfig(cached.files[pxt.CONFIG_NAME]) if (cfg) { @@ -630,10 +639,28 @@ namespace pxt.github { return { version, config }; } - export async function cacheProjectDependenciesAsync(cfg: pxt.PackageConfig): Promise { + export async function cacheProjectDependenciesAsync( + cfg: pxt.PackageConfig, + backupExtensions?: pxt.Map> + ): Promise { + return cacheProjectDependenciesAsyncCore( + cfg, + {} /** resolved */, + backupExtensions + ); + } + + async function cacheProjectDependenciesAsyncCore( + cfg: pxt.PackageConfig, + checked: pxt.Map, + backupExtensions?: pxt.Map> + ): Promise { const ghExtensions = Object.keys(cfg.dependencies) ?.filter(dep => isGithubId(cfg.dependencies[dep])); + // need to check/cache pub: links + inject to cache? + // probably fine to put off from initial pass as it's a bit of an edge case? + // maybe extend pxt.github.cache... a bit? if (ghExtensions.length) { const pkgConfig = await pxt.packagesConfigAsync(); // Make sure external packages load before installing header. @@ -641,10 +668,17 @@ namespace pxt.github { ghExtensions.map( async ext => { const extSrc = cfg.dependencies[ext]; - const ghPkg = await downloadPackageAsync(extSrc, pkgConfig); + if (checked[extSrc]) + return; + checked[extSrc] = true; + const backup = backupExtensions?.[extSrc]; + const ghPkg = await downloadPackageAsync(extSrc, pkgConfig, backup); if (!ghPkg) { throw new Error(lf("Cannot load extension {0} from {1}", ext, extSrc)); } + + const pkgCfg = pxt.U.jsonTryParse(ghPkg.files[pxt.CONFIG_NAME]); + await cacheProjectDependenciesAsyncCore(pkgCfg, checked, backupExtensions); } ) ); diff --git a/pxtlib/main.ts b/pxtlib/main.ts index 81451cc05338..21941c50a0c9 100644 --- a/pxtlib/main.ts +++ b/pxtlib/main.ts @@ -512,6 +512,9 @@ namespace pxt { export const TUTORIAL_CUSTOM_TS = "tutorial.custom.ts"; export const BREAKPOINT_TABLET = 991; // TODO (shakao) revisit when tutorial stuff is more settled export const PALETTES_FILE = "_palettes.json"; + // for packing extensions into distributables, as backup when network unavailable + export const PACKAGED_EXTENSIONS = "_packaged-extensions.json"; + export const PACKAGED_EXT_INFO = "_packaged-ext-info.json"; export function outputName(trg: pxtc.CompileTarget = null) { if (!trg) trg = appTarget.compile diff --git a/pxtlib/package.ts b/pxtlib/package.ts index 68bec09d7ab3..72a29c27fd5a 100644 --- a/pxtlib/package.ts +++ b/pxtlib/package.ts @@ -1220,8 +1220,7 @@ namespace pxt { variants = [null] } - - let ext: pxtc.ExtensionInfo = null + let ext: pxtc.ExtensionInfo = null; for (let v of variants) { if (ext) pxt.debug(`building for ${v}`) @@ -1247,7 +1246,16 @@ namespace pxt { !opts.target.isNative if (!noFileEmbed) { - const files = await this.filesToBePublishedAsync(true) + // Include packages when it won't influence flash size. + const files = await this.filesToBePublishedAsync( + true, + !appTarget.compile.useUF2 + ); + if (opts.target.isNative && opts.extinfo.hexinfo) { + // todo trim down to relevant portion of extinfo? + // hexfile + hash + whatever is needed for /cpp.ts + files[pxt.PACKAGED_EXT_INFO] = JSON.stringify(opts.extinfo); + } const headerString = JSON.stringify({ name: this.config.name, comment: this.config.description, @@ -1257,7 +1265,9 @@ namespace pxt { editor: this.getPreferredEditor(), targetVersions: pxt.appTarget.versions }) - const programText = JSON.stringify(files) + + const programText = JSON.stringify(files); + const buf = await lzmaCompressAsync(headerString + programText) if (buf) { opts.embedMeta = JSON.stringify({ @@ -1268,7 +1278,7 @@ namespace pxt { eURL: pxt.appTarget.appTheme.embedUrl, eVER: pxt.appTarget.versions ? pxt.appTarget.versions.target : "", pxtTarget: appTarget.id, - }) + }); opts.embedBlob = ts.pxtc.encodeBase64(U.uint8ArrayToString(buf)) } } @@ -1314,24 +1324,57 @@ namespace pxt { return cfg; } - filesToBePublishedAsync(allowPrivate = false) { + async filesToBePublishedAsync(allowPrivate = false, packExternalExtensions = false) { const files: Map = {}; - return this.loadAsync() - .then(() => { - if (!allowPrivate && !this.config.public) - U.userError('Only packages with "public":true can be published') - const cfg = this.prepareConfigToBePublished(); - files[pxt.CONFIG_NAME] = pxt.Package.stringifyConfig(cfg); - for (let f of this.getFiles()) { - // already stored - if (f == pxt.CONFIG_NAME) continue; - let str = this.readFile(f) - if (str == null) - U.userError("referenced file missing: " + f) - files[f] = str + await this.loadAsync(); + if (!allowPrivate && !this.config.public) + U.userError('Only packages with "public":true can be published') + const cfg = this.prepareConfigToBePublished(); + files[pxt.CONFIG_NAME] = pxt.Package.stringifyConfig(cfg); + + for (let f of this.getFiles()) { + // already stored + if (f == pxt.CONFIG_NAME) continue; + let str = this.readFile(f) + if (str == null) + U.userError("referenced file missing: " + f) + files[f] = str + } + + if (packExternalExtensions) { + const packedDeps: Map> = {}; + const packDeps = (p: Package) => { + const depsToPack = p.resolvedDependencies() + .filter(dep => { + switch (dep.verProtocol()) { + case "github": + case "pub": + return true; + default: + return false; + } + }); + + for (const dep of depsToPack) { + if (packedDeps[dep._verspec]) + continue; + const packed: Map = {}; + for (const toPack of dep.getFiles()) { + packed[toPack] = dep.readFile(toPack); + } + packed[pxt.CONFIG_NAME] = JSON.stringify(dep.config); + + packedDeps[dep._verspec] = packed; + packDeps(dep); } - return U.sortObjectFields(files) - }) + } + + packDeps(this); + if (Object.keys(packedDeps).length) { + files[pxt.PACKAGED_EXTENSIONS] = JSON.stringify(packedDeps); + } + } + return U.sortObjectFields(files); } saveToJsonAsync(): Promise { diff --git a/webapp/src/db.ts b/webapp/src/db.ts index 4260ff18d985..b5fbdb11db8c 100644 --- a/webapp/src/db.ts +++ b/webapp/src/db.ts @@ -122,32 +122,35 @@ class GithubDb implements pxt.github.IGithubDb { } // not found ); } - loadPackageAsync(repopath: string, tag: string): Promise { + async loadPackageAsync(repopath: string, tag: string, backupScriptText?: pxt.Map): Promise { if (!tag) { - pxt.debug(`dep: default to master`) - tag = "master" + pxt.debug(`dep: default to master`) + tag = "master" } // don't cache master if (tag == "master") - return this.mem.loadPackageAsync(repopath, tag); + return this.mem.loadPackageAsync(repopath, tag, backupScriptText); const id = `pkg-${repopath}-${tag}`; - return this.table.getAsync(id).then( - entry => { - pxt.debug(`github offline cache hit ${id}`); - return entry.package as pxt.github.CachedPackage; - }, - e => { - pxt.debug(`github offline cache miss ${id}`); - return this.mem.loadPackageAsync(repopath, tag) - .then(p => { - return this.table.forceSetAsync({ - id, - package: p - }).then(() => p, e => p); - }) - } // not found - ); + try { + const entry = await this.table.getAsync(id); + pxt.debug(`github offline cache hit ${id}`); + // TODO: back up check here if .backupCopy to try fetch from this.mem? + return entry.package as pxt.github.CachedPackage; + + } catch (e) { + pxt.debug(`github offline cache miss ${id}`); + const p = await this.mem.loadPackageAsync(repopath, tag, backupScriptText); + try { + await this.table.forceSetAsync({ + id, + package: p + }); + } catch (e) { + // swallow caching error + } + return p; + } } } diff --git a/webapp/src/workspace.ts b/webapp/src/workspace.ts index bfbde0a7a5fb..6acdc5212040 100644 --- a/webapp/src/workspace.ts +++ b/webapp/src/workspace.ts @@ -646,7 +646,24 @@ export function installAsync(h0: InstallHeader, text: ScriptText, dontOverwriteI pxt.shell.setEditorLanguagePref(cfg.preferredEditor); } - return pxt.github.cacheProjectDependenciesAsync(cfg) + let backupExtensionFiles: pxt.Map>; + if (text[pxt.PACKAGED_EXTENSIONS]) { + backupExtensionFiles = pxt.Util.jsonTryParse(text[pxt.PACKAGED_EXTENSIONS]); + // TODO cache hexfiles in text[pxt.PACKAGED_HEXFILE] like field? + // Would be necessary for full offline usage, but tricky as it + // also depends on target version / hex hash; + + // Do not persist into project once installed. + delete text[pxt.PACKAGED_EXTENSIONS]; + } + if (text[pxt.PACKAGED_EXT_INFO]) { + const parsedExtensionInfo = pxt.Util.jsonTryParse(text[pxt.PACKAGED_EXT_INFO]); + // TODO: push this into cache; see pxtlib/cpp.ts, would need to mirror portion of + // getHexInfoAsync that compresses hex download result and pushes into storeWithLimitAsync + delete text[pxt.PACKAGED_EXT_INFO]; + } + + return pxt.github.cacheProjectDependenciesAsync(cfg, backupExtensionFiles) .then(() => importAsync(h, text)) .then(() => h); }