From 75706b0ccb6b59e12c96c295f324ef549820adfe Mon Sep 17 00:00:00 2001 From: Richard Knoll Date: Tue, 30 Apr 2024 14:06:21 -0700 Subject: [PATCH 1/4] [stable9.0] Cherry picking crowdin changes over to Calliope's stable branch (#9991) * [V9] Cherry picking crowdin work over to v9 branch (#9963) * Upgrade to Crowdin V2 APIs (#9954) * use crowdin v2 apis * rename crowdin API * add projectid setting * fix test mode * copy pxt-translations crowdin config * re-enable crowdin stats (#9962) * Add restore command and keep translations when updating (#9966) (#9968) * add restore command and keep translations when updating * add --test flag * add parameter to prevent accidental runs --- cli/cli.ts | 107 ++++---- cli/crowdin.ts | 492 +++++++++++++++++++++---------------- cli/crowdinApi.ts | 465 +++++++++++++++++++++++++++++++++++ localtypings/pxtarget.d.ts | 1 + package.json | 4 + pxtlib/crowdin.ts | 284 --------------------- 6 files changed, 808 insertions(+), 545 deletions(-) create mode 100644 cli/crowdinApi.ts diff --git a/cli/cli.ts b/cli/cli.ts index a4251b627563..7889b4408b37 100644 --- a/cli/cli.ts +++ b/cli/cli.ts @@ -393,7 +393,7 @@ function checkIfTaggedCommitAsync() { let readJson = nodeutil.readJson; -function ciAsync() { +async function ciAsync() { forceCloudBuild = true; const buildInfo = ciBuildInfo(); pxt.log(`ci build using ${buildInfo.ci}`); @@ -446,53 +446,53 @@ function ciAsync() { let pkg = readJson("package.json") if (pkg["name"] == "pxt-core") { pxt.log("pxt-core build"); - return checkIfTaggedCommitAsync() - .then(isTaggedCommit => { - pxt.log(`is tagged commit: ${isTaggedCommit}`); - let p = npmPublishAsync(); - if (branch === "master" && isTaggedCommit) { - if (uploadDocs) - p = p - .then(() => buildWebStringsAsync()) - .then(() => crowdin.execCrowdinAsync("upload", "built/webstrings.json")) - .then(() => crowdin.execCrowdinAsync("upload", "built/skillmap-strings.json")) - .then(() => crowdin.execCrowdinAsync("upload", "built/authcode-strings.json")) - .then(() => crowdin.execCrowdinAsync("upload", "built/multiplayer-strings.json")) - .then(() => crowdin.execCrowdinAsync("upload", "built/kiosk-strings.json")); - if (uploadApiStrings) - p = p.then(() => crowdin.execCrowdinAsync("upload", "built/strings.json")) - if (uploadDocs || uploadApiStrings) - p = p.then(() => crowdin.internalUploadTargetTranslationsAsync(uploadApiStrings, uploadDocs)); - } - return p; - }); + + const isTaggedCommit = await checkIfTaggedCommitAsync(); + pxt.log(`is tagged commit: ${isTaggedCommit}`); + await npmPublishAsync(); + if (branch === "master" && isTaggedCommit) { + if (uploadDocs) { + await buildWebStringsAsync(); + await crowdin.uploadBuiltStringsAsync("built/webstrings.json"); + await crowdin.uploadBuiltStringsAsync(`built/skillmap-strings.json`); + await crowdin.uploadBuiltStringsAsync(`built/authcode-strings.json`); + await crowdin.uploadBuiltStringsAsync(`built/multiplayer-strings.json`); + await crowdin.uploadBuiltStringsAsync(`built/kiosk-strings.json`); + } + if (uploadApiStrings) { + await crowdin.uploadBuiltStringsAsync("built/strings.json"); + } + if (uploadDocs || uploadApiStrings) { + await crowdin.internalUploadTargetTranslationsAsync(uploadApiStrings, uploadDocs); + pxt.log("translations uploaded"); + } + else { + pxt.log("skipping translations upload"); + } + } } else { pxt.log("target build"); - return internalBuildTargetAsync() - .then(() => internalCheckDocsAsync(true)) - .then(() => blockTestsAsync()) - .then(() => npmPublishAsync()) - .then(() => { - if (!process.env["PXT_ACCESS_TOKEN"]) { - // pull request, don't try to upload target - pxt.log('no token, skipping upload') - return Promise.resolve(); - } - const trg = readLocalPxTarget(); - const label = `${trg.id}/${tag || latest}`; - pxt.log(`uploading target with label ${label}...`); - return uploadTargetAsync(label); - }) - .then(() => { - pxt.log("target uploaded"); - if (uploadDocs || uploadApiStrings) { - return crowdin.internalUploadTargetTranslationsAsync(uploadApiStrings, uploadDocs) - .then(() => pxt.log("translations uploaded")); - } else { - pxt.log("skipping translations upload"); - return Promise.resolve(); - } - }); + await internalBuildTargetAsync(); + await internalCheckDocsAsync(true); + await blockTestsAsync(); + await npmPublishAsync(); + if (!process.env["PXT_ACCESS_TOKEN"]) { + // pull request, don't try to upload target + pxt.log('no token, skipping upload') + } + else { + const trg = readLocalPxTarget(); + const label = `${trg.id}/${tag || latest}`; + pxt.log(`uploading target with label ${label}...`); + await uploadTargetAsync(label); + } + pxt.log("target uploaded"); + if (uploadDocs || uploadApiStrings) { + await crowdin.internalUploadTargetTranslationsAsync(uploadApiStrings, uploadDocs); + pxt.log("translations uploaded"); + } else { + pxt.log("skipping translations upload"); + } } } @@ -7017,7 +7017,7 @@ ${pxt.crowdin.KEY_VARIABLE} - crowdin key advanced: true, }, pc => uploadTargetRefsAsync(pc.args[0])); advancedCommand("uploadtt", "upload tagged release", uploadTaggedTargetAsync, ""); - advancedCommand("downloadtrgtranslations", "download translations from bundled projects", crowdin.downloadTargetTranslationsAsync, ""); + advancedCommand("downloadtrgtranslations", "download translations from bundled projects", crowdin.downloadTargetTranslationsAsync, "[package]"); p.defineCommand({ name: "checkdocs", @@ -7103,7 +7103,18 @@ ${pxt.crowdin.KEY_VARIABLE} - crowdin key advancedCommand("augmentdocs", "test markdown docs replacements", augmnetDocsAsync, " "); - advancedCommand("crowdin", "upload, download, clean, stats files to/from crowdin", pc => crowdin.execCrowdinAsync.apply(undefined, pc.args), " [output]") + p.defineCommand({ + name: "crowdin", + advanced: true, + argString: " [output]", + help: "upload, download, clean, stats files to/from crowdin", + flags: { + test: { description: "test run, do not upload files to crowdin" } + } + }, pc => { + if (pc.flags.test) pxt.crowdin.setTestMode(); + return crowdin.execCrowdinAsync.apply(undefined, pc.args) + }) advancedCommand("hidlist", "list HID devices", hid.listAsync) advancedCommand("hidserial", "run HID serial forwarding", hid.serialAsync, undefined, true); diff --git a/cli/crowdin.ts b/cli/crowdin.ts index 39ccd7713cbd..7e2d4255966a 100644 --- a/cli/crowdin.ts +++ b/cli/crowdin.ts @@ -5,109 +5,87 @@ import * as path from 'path'; import Map = pxt.Map; import * as commandParser from './commandparser'; - -interface CrowdinCredentials { prj: string; key: string; branch: string; } - -function crowdinCredentialsAsync(): Promise { - const prj = pxt.appTarget.appTheme.crowdinProject; - const branch = pxt.appTarget.appTheme.crowdinBranch; - if (!prj) { - pxt.log(`crowdin upload skipped, Crowdin project missing in target theme`); - return Promise.resolve(undefined); - } - - let key: string; - if (pxt.crowdin.testMode) - key = pxt.crowdin.TEST_KEY; - else - key = process.env[pxt.crowdin.KEY_VARIABLE]; - if (!key) { - pxt.log(`Crowdin operation skipped: '${pxt.crowdin.KEY_VARIABLE}' variable is missing`); - return Promise.resolve(undefined); - } - return Promise.resolve({ prj, key, branch }); -} +import { downloadFileTranslationsAsync, getFileProgressAsync, listFilesAsync, restoreFileBefore, uploadFileAsync } from './crowdinApi'; export function uploadTargetTranslationsAsync(parsed?: commandParser.ParsedCommand) { const uploadDocs = parsed && !!parsed.flags["docs"]; const uploadApiStrings = parsed && !!parsed.flags["apis"] - if (parsed && !!parsed.flags["test"]) + if (parsed && !!parsed.flags["test"]) { pxt.crowdin.setTestMode(); + } return internalUploadTargetTranslationsAsync(uploadApiStrings, uploadDocs); } -export function internalUploadTargetTranslationsAsync(uploadApiStrings: boolean, uploadDocs: boolean) { +export async function internalUploadTargetTranslationsAsync(uploadApiStrings: boolean, uploadDocs: boolean) { pxt.log(`uploading translations (apis ${uploadApiStrings ? "yes" : "no"}, docs ${uploadDocs ? "yes" : "no"})...`); - return crowdinCredentialsAsync() - .then(cred => { - if (!cred) return Promise.resolve(); - pxt.log("got Crowdin credentials"); - const crowdinDir = pxt.appTarget.id; - if (crowdinDir == "core") { - if (!uploadDocs) { - pxt.log('missing --docs flag, skipping') - return Promise.resolve(); - } - return uploadDocsTranslationsAsync("docs", crowdinDir, cred.branch, cred.prj, cred.key) - .then(() => uploadDocsTranslationsAsync("common-docs", crowdinDir, cred.branch, cred.prj, cred.key)) - } else { - let p = Promise.resolve(); - if (uploadApiStrings) - p = p.then(() => execCrowdinAsync("upload", "built/target-strings.json", crowdinDir)) - .then(() => fs.existsSync("built/sim-strings.json") ? execCrowdinAsync("upload", "built/sim-strings.json", crowdinDir) : Promise.resolve()) - .then(() => uploadBundledTranslationsAsync(crowdinDir, cred.branch, cred.prj, cred.key)); - else - p = p.then(() => pxt.log(`translations: skipping api strings upload`)); - if (uploadDocs) - p = p.then(() => uploadDocsTranslationsAsync("docs", crowdinDir, cred.branch, cred.prj, cred.key)) - // scan for docs in bundled packages - .then(() => Promise.all(pxt.appTarget.bundleddirs - // there must be a folder under .../docs - .filter(pkgDir => nodeutil.existsDirSync(path.join(pkgDir, "docs"))) - // upload to crowdin - .map(pkgDir => uploadDocsTranslationsAsync(path.join(pkgDir, "docs"), crowdinDir, cred.branch, cred.prj, cred.key) - )).then(() => { - pxt.log("docs uploaded"); - })); - else - p = p.then(() => pxt.log(`translations: skipping docs upload`)); - return p; + + const crowdinDir = pxt.appTarget.id; + if (crowdinDir == "core") { + if (!uploadDocs) { + pxt.log('missing --docs flag, skipping') + return; + } + + await uploadDocsTranslationsAsync("docs", crowdinDir); + await uploadDocsTranslationsAsync("common-docs", crowdinDir); + } else { + if (uploadApiStrings) { + await uploadBuiltStringsAsync("built/target-strings.json", crowdinDir); + + if (fs.existsSync("built/sim-strings.json")) { + await uploadBuiltStringsAsync("built/sim-strings.json", crowdinDir); } - }); + await uploadBundledTranslationsAsync(crowdinDir); + } + else { + pxt.log(`translations: skipping api strings upload`); + } + + if (uploadDocs) { + await uploadDocsTranslationsAsync("docs", crowdinDir); + await Promise.all( + pxt.appTarget.bundleddirs + .filter(pkgDir => nodeutil.existsDirSync(path.join(pkgDir, "docs"))) + .map(pkgDir => uploadDocsTranslationsAsync(path.join(pkgDir, "docs"), crowdinDir)) + ); + pxt.log("docs uploaded"); + } + else { + pxt.log(`translations: skipping docs upload`); + } + } } -function uploadDocsTranslationsAsync(srcDir: string, crowdinDir: string, branch: string, prj: string, key: string): Promise { - pxt.log(`uploading from ${srcDir} to ${crowdinDir} under project ${prj}/${branch || ""}`) + +export async function uploadBuiltStringsAsync(filename: string, crowdinDir?: string) { + const baseName = path.basename(filename); + const crowdinFile = crowdinDir ? path.join(crowdinDir, baseName) : baseName; + const contents = fs.readFileSync(filename, "utf8"); + + pxt.log(`Uploading ${filename} to ${crowdinFile}`); + await uploadFileAsync(crowdinFile, contents); +} + +async function uploadDocsTranslationsAsync(srcDir: string, crowdinDir: string): Promise { + pxt.log(`Uploading from ${srcDir} to ${crowdinDir}`) const ignoredDirectoriesList = getIgnoredDirectories(srcDir); const todo = nodeutil.allFiles(srcDir).filter(f => /\.md$/.test(f) && !/_locales/.test(f)).reverse(); - const knownFolders: Map = {}; - const ensureFolderAsync = (crowdd: string) => { - if (!knownFolders[crowdd]) { - knownFolders[crowdd] = true; - pxt.log(`creating folder ${crowdd}`); - return pxt.crowdin.createDirectoryAsync(branch, prj, key, crowdd); - } - return Promise.resolve(); - } - const nextFileAsync = (f: string): Promise => { - if (!f) return Promise.resolve(); - const crowdf = path.join(crowdinDir, f); - const crowdd = path.dirname(crowdf); + + for (const file of todo) { + if (!file) continue; + + const crowdinFile = path.join(crowdinDir, file); + // check if file should be ignored - if (ignoredDirectoriesList.filter(d => path.dirname(f).indexOf(d) == 0).length > 0) { - pxt.log(`skipping ${f} because of .crowdinignore file`) - return nextFileAsync(todo.pop()); + if (ignoredDirectoriesList.filter(d => path.dirname(file).indexOf(d) == 0).length > 0) { + pxt.log(`skipping ${file} because of .crowdinignore file`) + continue; } - const data = fs.readFileSync(f, 'utf8'); - pxt.log(`uploading ${f} to ${crowdf}`); - return ensureFolderAsync(crowdd) - .then(() => pxt.crowdin.uploadTranslationAsync(branch, prj, key, crowdf, data)) - .then(() => nextFileAsync(todo.pop())); + pxt.log(`Uploading ${file} to ${crowdinFile}`); + await uploadFileAsync(crowdinFile, fs.readFileSync(file, "utf8")); } - return ensureFolderAsync(path.join(crowdinDir, srcDir)) - .then(() => nextFileAsync(todo.pop())); } function getIgnoredDirectories(srcDir: string) { @@ -125,43 +103,46 @@ function getIgnoredDirectories(srcDir: string) { return Object.keys(ignoredDirectories).filter(d => ignoredDirectories[d]); } -function uploadBundledTranslationsAsync(crowdinDir: string, branch: string, prj: string, key: string): Promise { +async function uploadBundledTranslationsAsync(crowdinDir: string) { const todo: string[] = []; - pxt.appTarget.bundleddirs.forEach(dir => { + for (const dir of pxt.appTarget.bundleddirs) { const locdir = path.join(dir, "_locales"); - if (fs.existsSync(locdir)) - fs.readdirSync(locdir) - .filter(f => /strings\.json$/i.test(f)) - .forEach(f => todo.push(path.join(locdir, f))); - }); + if (fs.existsSync(locdir)) { + const stringsFiles = fs.readdirSync(locdir).filter(f => /strings\.json$/i.test(f)); + + for (const file of stringsFiles) { + todo.unshift(path.join(locdir, file)); + } + } + } pxt.log(`uploading bundled translations to Crowdin (${todo.length} files)`); - const nextFileAsync = (): Promise => { - const f = todo.pop(); - if (!f) return Promise.resolve(); - const data = JSON.parse(fs.readFileSync(f, 'utf8')) as Map; - const crowdf = path.join(crowdinDir, path.basename(f)); - pxt.log(`uploading ${f} to ${crowdf}`); - return pxt.crowdin.uploadTranslationAsync(branch, prj, key, crowdf, JSON.stringify(data)) - .then(nextFileAsync); + for (const file of todo) { + const data = JSON.parse(fs.readFileSync(file, 'utf8')) as Map; + const crowdinFile = path.join(crowdinDir, path.basename(file)); + pxt.log(`Uploading ${file} to ${crowdinFile}`); + await uploadFileAsync(crowdinFile, JSON.stringify(data)); } - return nextFileAsync(); } export async function downloadTargetTranslationsAsync(parsed?: commandParser.ParsedCommand) { const name = parsed?.args[0]; - const cred = await crowdinCredentialsAsync(); - - if (!cred) - return; await buildAllTranslationsAsync(async (fileName: string) => { pxt.log(`downloading ${fileName}`); - return pxt.crowdin.downloadTranslationsAsync(cred.branch, cred.prj, cred.key, fileName, { translatedOnly: true, validatedOnly: true }); + + const translations = await downloadFileTranslationsAsync(fileName); + const parsed: pxt.Map> = {}; + + for (const file of Object.keys(translations)) { + parsed[file] = JSON.parse(translations[file]); + } + + return parsed; }, name); } -export async function buildAllTranslationsAsync(langToStringsHandlerAsync: (fileName: string) => Promise>>, singleDir?: string) { +export async function buildAllTranslationsAsync(fetchFileTranslationAsync: (fileName: string) => Promise>>, singleDir?: string) { await buildTranslationFilesAsync(["sim-strings.json"], "sim-strings.json"); await buildTranslationFilesAsync(["target-strings.json"], "target-strings.json"); await buildTranslationFilesAsync(["strings.json"], "strings.json", true); @@ -190,7 +171,7 @@ export async function buildAllTranslationsAsync(langToStringsHandlerAsync: (file const locdir = path.dirname(filePath); const projectdir = path.dirname(locdir); pxt.debug(`projectdir: ${projectdir}`); - const data = await langToStringsHandlerAsync(crowdf); + const data = await fetchFileTranslationAsync(crowdf); for (const lang of Object.keys(data)) { const dataLang = data[lang]; @@ -226,121 +207,206 @@ function stringifyTranslations(strings: pxt.Map): string { else return JSON.stringify(trg, null, 2); } -export function execCrowdinAsync(cmd: string, ...args: string[]): Promise { +export async function execCrowdinAsync(cmd: string, ...args: string[]): Promise { pxt.log(`executing Crowdin command ${cmd}...`); - const prj = pxt.appTarget.appTheme.crowdinProject; - if (!prj) { - console.log(`crowdin operation skipped, crowdin project not specified in pxtarget.json`); - return Promise.resolve(); - } - const branch = pxt.appTarget.appTheme.crowdinBranch; - return crowdinCredentialsAsync() - .then(crowdinCredentials => { - if (!crowdinCredentials) return Promise.resolve(); - const key = crowdinCredentials.key; - cmd = cmd.toLowerCase(); - if (!args[0] && (cmd != "clean" && cmd != "stats")) throw new Error(cmd == "status" ? "language missing" : "filename missing"); - switch (cmd) { - case "stats": return statsCrowdinAsync(prj, key, args[0]); - case "clean": return cleanCrowdinAsync(prj, key, args[0] || "docs"); - case "upload": return uploadCrowdinAsync(branch, prj, key, args[0], args[1]); - case "download": { - if (!args[1]) throw new Error("output path missing"); - const fn = path.basename(args[0]); - return pxt.crowdin.downloadTranslationsAsync(branch, prj, key, args[0], { translatedOnly: true, validatedOnly: true }) - .then(r => { - Object.keys(r).forEach(k => { - const rtranslations = stringifyTranslations(r[k]); - if (!rtranslations) return; - - nodeutil.mkdirP(path.join(args[1], k)); - const outf = path.join(args[1], k, fn); - console.log(`writing ${outf}`) - nodeutil.writeFileSync( - outf, - rtranslations, - { encoding: "utf8" }); - }) - }) - } - default: throw new Error("unknown command"); + + switch (cmd.toLowerCase()) { + case "stats": + execStatsAsync(args[0]); + break; + case "clean": + await execCleanAsync(args[0] || "docs"); + break; + case "upload": + if (!args[0]) { + throw new Error("filename missing"); + } + await uploadBuiltStringsAsync(args[0], args[1]); + break; + case "download": + if (!args[1]) { + throw new Error("output path missing"); + } + await execDownloadAsync(args[0], args[1]); + break; + case "restore": + if (!args[0]) { + throw new Error("Time missing"); + } + if (args[1] !== "force" && !pxt.crowdin.testMode) { + throw new Error(`Refusing to run restore command without 'force' argument. Re-run as 'pxt crowdin restore force' to proceed or use --test flag to test.`); } - }) + execRestoreFiles(args[0]); + break; + default: + throw new Error("unknown command"); + } } -function cleanCrowdinAsync(prj: string, key: string, dir: string): Promise { - const p = pxt.appTarget.id + "/" + dir; - return pxt.crowdin.listFilesAsync(prj, key, p) - .then(files => { - files.filter(f => !nodeutil.fileExistsSync(f.fullName.substring(pxt.appTarget.id.length + 1))) - .forEach(f => pxt.log(`crowdin: dead file: ${f.branch ? f.branch + "/" : ""}${f.fullName}`)); - }) +async function execDownloadAsync(filename: string, outputDir: string): Promise { + const basename = path.basename(filename); + pxt.log("Downloading translations") + const translations = await downloadFileTranslationsAsync(filename); + + for (const language of Object.keys(translations)) { + const langTranslations = stringifyTranslations(JSON.parse(translations[language])); + if (!langTranslations) continue; + + nodeutil.mkdirP(path.join(outputDir, language)); + const outFilename = path.join(outputDir, language, basename); + console.log(`Writing ${outFilename}`); + nodeutil.writeFileSync(outFilename, langTranslations, { encoding: "utf8" }); + } } -function statsCrowdinAsync(prj: string, key: string, preferredLang?: string): Promise { - pxt.log(`collecting crowdin stats for ${prj} ${preferredLang ? `for language ${preferredLang}` : `all languages`}`); - console.log(`context\t language\t translated%\t approved%\t phrases\t translated\t approved`) +async function execCleanAsync(dir: string): Promise { + const directoryPath = pxt.appTarget.id + "/" + dir; - const fn = `crowdinstats.csv`; - let headers = 'sep=\t\r\n'; - headers += `id\t file\t language\t phrases\t translated\t approved\r\n`; - nodeutil.writeFileSync(fn, headers, { encoding: "utf8" }); - return pxt.crowdin.projectInfoAsync(prj, key) - .then(info => { - if (!info) throw new Error("info failed") - let languages = info.languages; - // remove in-context language - languages = languages.filter(l => l.code != ts.pxtc.Util.TRANSLATION_LOCALE); - if (preferredLang) - languages = languages.filter(lang => lang.code.toLowerCase() == preferredLang.toLowerCase()); - return Promise.all(languages.map(lang => langStatsCrowdinAsync(prj, key, lang.code))) - }).then(() => { - console.log(`stats written to ${fn}`) - }) - - function langStatsCrowdinAsync(prj: string, key: string, lang: string): Promise { - return pxt.crowdin.languageStatsAsync(prj, key, lang) - .then(stats => { - let uiphrases = 0; - let uitranslated = 0; - let uiapproved = 0; - let corephrases = 0; - let coretranslated = 0; - let coreapproved = 0; - let phrases = 0; - let translated = 0; - let approved = 0; - let r = ''; - stats.forEach(stat => { - const cfn = `${stat.branch ? stat.branch + "/" : ""}${stat.fullName}`; - r += `${stat.id}\t ${cfn}\t ${lang}\t ${stat.phrases}\t ${stat.translated}\t ${stat.approved}\r\n`; - if (stat.fullName == "strings.json") { - uiapproved += Number(stat.approved); - uitranslated += Number(stat.translated); - uiphrases += Number(stat.phrases); - } else if (/core-strings\.json$/.test(stat.fullName)) { - coreapproved += Number(stat.approved); - coretranslated += Number(stat.translated); - corephrases += Number(stat.phrases); - } else if (/-strings\.json$/.test(stat.fullName)) { - approved += Number(stat.approved); - translated += Number(stat.translated); - phrases += Number(stat.phrases); - } - }) - fs.appendFileSync(fn, r, { encoding: "utf8" }); - console.log(`ui\t ${lang}\t ${(uitranslated / uiphrases * 100) >> 0}%\t ${(uiapproved / uiphrases * 100) >> 0}%\t ${uiphrases}\t ${uitranslated}\t ${uiapproved}`) - console.log(`core\t ${lang}\t ${(coretranslated / corephrases * 100) >> 0}%\t ${(coreapproved / corephrases * 100) >> 0}%\t ${corephrases}\t ${coretranslated}\t ${coreapproved}`) - console.log(`blocks\t ${lang}\t ${(translated / phrases * 100) >> 0}%\t ${(approved / phrases * 100) >> 0}%\t ${phrases}\t ${translated}\t ${approved}`) - }) + const files = await listFilesAsync(directoryPath); + + for (const file of files) { + if (!nodeutil.fileExistsSync(file.substring(pxt.appTarget.id.length + 1))) { + pxt.log(`crowdin: dead file: ${file}`) + } } +} +interface LanguageStats { + uiphrases: number; + uitranslated: number; + uiapproved: number; + corephrases: number; + coretranslated: number; + coreapproved: number; + phrases: number; + translated: number; + approved: number; } -function uploadCrowdinAsync(branch: string, prj: string, key: string, p: string, dir?: string): Promise { - let fn = path.basename(p); - if (dir) fn = dir.replace(/[\\/]*$/g, '') + '/' + fn; - const data = JSON.parse(fs.readFileSync(p, "utf8")) as Map; - pxt.log(`upload ${fn} (${Object.keys(data).length} strings) to https://crowdin.com/project/${prj}${branch ? `?branch=${branch}` : ''}`); - return pxt.crowdin.uploadTranslationAsync(branch, prj, key, fn, JSON.stringify(data)); +async function execStatsAsync(language?: string) { + const crowdinDir = pxt.appTarget.id; + + // If this is run inside pxt-core, give results for all targets + const isCore = crowdinDir === "core"; + pxt.log(`collecting crowdin stats for ${isCore ? "all targets" : crowdinDir} ${language ? `for language ${language}` : `all languages`}`); + + const files = await listFilesAsync(); + const stats: pxt.Map = {}; + + const outputCsvFile = `crowdinstats.csv`; + let headers = 'sep=\t\r\n'; + headers += `file\t language\t phrases\t translated\t approved\r\n`; + nodeutil.writeFileSync(outputCsvFile, headers, { encoding: "utf8" }); + + for (const file of files) { + pxt.debug("Processing file: " + file + "..."); + + // We only care about strings files + if (!file.endsWith("-strings.json")) continue; + + // Files for core are in the top-level of the crowdin project + const isCoreFile = file.indexOf("/") === -1; + + // Only include files for the current target and core + if (!isCore && !isCoreFile && !file.startsWith(crowdinDir + "/")) continue; + + pxt.debug(`Downloading progress`) + const progress = await getFileProgressAsync(file, language && [language]); + + let fileCsvRows = ""; + for (const language of progress) { + if (!stats[language.languageId]) { + stats[language.languageId] = { + uiphrases: 0, + uitranslated: 0, + uiapproved: 0, + corephrases: 0, + coretranslated: 0, + coreapproved: 0, + phrases: 0, + translated: 0, + approved: 0 + }; + } + + const fileCsvColumns = [ + file, + language.languageId, + language.phrases.total, + language.phrases.translated, + language.phrases.approved + ]; + + fileCsvRows += `${fileCsvColumns.join("\t ")}\r\n`; + + const langStats = stats[language.languageId]; + + if (file === "strings.json") { + langStats.uiapproved += language.phrases.approved + langStats.uitranslated += language.phrases.translated; + langStats.uiphrases += language.phrases.total; + } + else if (/core-strings\.json$/.test(file)) { + langStats.coreapproved += language.phrases.approved + langStats.coretranslated += language.phrases.translated; + langStats.corephrases += language.phrases.total; + } + else { + langStats.approved += language.phrases.approved + langStats.translated += language.phrases.translated; + langStats.phrases += language.phrases.total; + } + } + + fs.appendFileSync(outputCsvFile, fileCsvRows, { encoding: "utf8" }); + } + + console.log(`context\t language\t translated%\t approved%\t phrases\t translated\t approved`) + for (const language of Object.keys(stats)) { + const { + uiphrases, + uitranslated, + uiapproved, + corephrases, + coretranslated, + coreapproved, + phrases, + translated, + approved, + } = stats[language]; + + console.log(`ui\t ${language}\t ${(uitranslated / uiphrases * 100) >> 0}%\t ${(uiapproved / uiphrases * 100) >> 0}%\t ${uiphrases}\t ${uitranslated}\t ${uiapproved}`) + console.log(`core\t ${language}\t ${(coretranslated / corephrases * 100) >> 0}%\t ${(coreapproved / corephrases * 100) >> 0}%\t ${corephrases}\t ${coretranslated}\t ${coreapproved}`) + console.log(`blocks\t ${language}\t ${(translated / phrases * 100) >> 0}%\t ${(approved / phrases * 100) >> 0}%\t ${phrases}\t ${translated}\t ${approved}`) + } } + +async function execRestoreFiles(time: string | number) { + let cutoffTime; + + if (!isNaN(parseInt(time + ""))) { + cutoffTime = parseInt(time + ""); + } + else { + cutoffTime = new Date(time).getTime(); + } + + const crowdinDir = pxt.appTarget.id; + + // If this is run inside pxt-core, give results for all targets + const isCore = crowdinDir === "core"; + + const files = await listFilesAsync(); + + for (const file of files) { + pxt.debug("Processing file: " + file + "..."); + + // Files for core are in the top-level of the crowdin project + const isCoreFile = file.indexOf("/") === -1; + + + if ((isCore && !isCoreFile) || !file.startsWith(crowdinDir + "/")) continue; + + await restoreFileBefore(file, cutoffTime); + } +} \ No newline at end of file diff --git a/cli/crowdinApi.ts b/cli/crowdinApi.ts new file mode 100644 index 000000000000..48547f178f06 --- /dev/null +++ b/cli/crowdinApi.ts @@ -0,0 +1,465 @@ +import crowdin, { Credentials, SourceFilesModel, ClientConfig } from '@crowdin/crowdin-api-client'; +import * as path from 'path'; +import axios from 'axios'; + +import * as AdmZip from "adm-zip"; + +let client: crowdin; +const KINDSCRIPT_PROJECT_ID = 157956; + +let projectId = KINDSCRIPT_PROJECT_ID; + +let fetchedFiles: SourceFilesModel.File[]; +let fetchedDirectories: SourceFilesModel.Directory[]; + +export function setProjectId(id: number) { + projectId = id; + fetchedFiles = undefined; + fetchedDirectories = undefined; +} + +export async function uploadFileAsync(fileName: string, fileContent: string): Promise { + if (pxt.crowdin.testMode) return; + + const files = await getAllFiles(); + + // If file already exists, update it + for (const file of files) { + if (normalizePath(file.path) === normalizePath(fileName)) { + await updateFile(file.id, path.basename(fileName), fileContent); + return; + } + } + + // Ensure directory exists + const parentDir = path.dirname(fileName); + let parentDirId: number; + if (parentDir && parentDir !== ".") { + parentDirId = (await mkdirAsync(parentDir)).id; + } + + // Create new file + await createFile(path.basename(fileName), fileContent, parentDirId); +} + +export async function getProjectInfoAsync() { + const { projectsGroupsApi } = getClient(); + + const project = await projectsGroupsApi.getProject(projectId); + return project.data; +} + +export async function getProjectProgressAsync(languages?: string[]) { + const { translationStatusApi } = getClient(); + + const stats = await translationStatusApi + .withFetchAll() + .getProjectProgress(projectId); + + let results = stats.data.map(stat => stat.data); + if (languages) { + results = results.filter(stat => languages.indexOf(stat.language.locale) !== -1 || languages.indexOf(stat.language.twoLettersCode) !== -1); + } + + return results; +} + +export async function getDirectoryProgressAsync(directory: string, languages?: string[]) { + const { translationStatusApi } = getClient(); + + const directoryId = await getDirectoryIdAsync(directory); + + const stats = await translationStatusApi + .withFetchAll() + .getDirectoryProgress(projectId, directoryId); + + let results = stats.data.map(stat => stat.data); + if (languages) { + results = results.filter(stat => languages.indexOf(stat.language.locale) !== -1 || languages.indexOf(stat.language.twoLettersCode) !== -1); + } + + return results; +} + +export async function getFileProgressAsync(file: string, languages?: string[]) { + const { translationStatusApi } = getClient(); + + const fileId = await getFileIdAsync(file); + + const stats = await translationStatusApi + .withFetchAll() + .getFileProgress(projectId, fileId); + + let results = stats.data.map(stat => stat.data); + if (languages) { + results = results.filter(stat => languages.indexOf(stat.language.locale) !== -1 || languages.indexOf(stat.language.twoLettersCode) !== -1); + } + + return results; +} + +export async function listFilesAsync(directory?: string): Promise { + const files = (await getAllFiles()).map(file => normalizePath(file.path)); + + if (directory) { + directory = normalizePath(directory); + return files.filter(file => file.startsWith(directory)); + } + return files; +} + +export async function downloadTranslationsAsync(directory?: string) { + const { translationsApi } = getClient(); + + let buildId: number; + let status: string; + + const options = { + skipUntranslatedFiles: true, + exportApprovedOnly: true + }; + + if (directory) { + pxt.log(`Building translations for directory ${directory}`); + const directoryId = await getDirectoryIdAsync(directory); + const buildResp = await translationsApi.buildProjectDirectoryTranslation(projectId, directoryId, options); + buildId = buildResp.data.id; + status = buildResp.data.status; + } + else { + pxt.log(`Building all translations`) + const buildResp = await translationsApi.buildProject(projectId, options); + buildId = buildResp.data.id; + status = buildResp.data.status; + } + + // Translation builds take a long time, so poll for progress + while (status !== "finished") { + const progress = await translationsApi.checkBuildStatus(projectId, buildId); + status = progress.data.status; + + pxt.log(`Translation build progress: ${progress.data.progress}%`) + if (status !== "finished") { + await pxt.Util.delay(5000); + } + } + + pxt.log("Fetching translation build"); + const downloadReq = await translationsApi.downloadTranslations(projectId, buildId); + + // The downloaded file is a zip of all files broken out in a directory for each language + // e.g. /en/docs/tutorial.md, /fr/docs/tutorial.md, etc. + pxt.log("Downloading translation zip"); + const zipFile = await axios.get(downloadReq.data.url, { responseType: 'arraybuffer' }); + + const zip = new AdmZip(Buffer.from(zipFile.data)); + + const entries = zip.getEntries(); + const filesystem: pxt.Map = {}; + + for (const entry of entries) { + if (entry.isDirectory) continue; + + filesystem[entry.entryName] = zip.readAsText(entry); + } + + pxt.log("Translation download complete"); + + return filesystem; +} + +export async function downloadFileTranslationsAsync(fileName: string): Promise> { + const { translationsApi } = getClient(); + const fileId = await getFileIdAsync(fileName); + const projectInfo = await getProjectInfoAsync(); + + let todo = projectInfo.targetLanguageIds.filter(id => id !== "en"); + + if (pxt.appTarget && pxt.appTarget.appTheme && pxt.appTarget.appTheme.availableLocales) { + todo = todo.filter(l => pxt.appTarget.appTheme.availableLocales.indexOf(l) > -1); + } + + const options = { + skipUntranslatedFiles: true, + exportApprovedOnly: true + }; + + const results: pxt.Map = {}; + + // There's no API to get all translations for a file, so we have to build each one individually + for (const language of todo) { + pxt.debug(`Building ${language} translation for '${fileName}'`); + + try { + const buildResp = await translationsApi.buildProjectFileTranslation(projectId, fileId, { + targetLanguageId: language, + ...options + }); + + if (!buildResp.data) { + pxt.debug(`No translation available for ${language}`); + continue; + } + + const textResp = await axios.get(buildResp.data.url, { responseType: "text" }); + results[language] = textResp.data; + } + catch (e) { + console.log(`Error building ${language} translation for '${fileName}'`, e); + continue; + } + } + + return results; +} + +async function getFileIdAsync(fileName: string): Promise { + for (const file of await getAllFiles()) { + if (normalizePath(file.path) === normalizePath(fileName)) { + return file.id; + } + } + + throw new Error(`File '${fileName}' not found in crowdin project`); +} + +async function getDirectoryIdAsync(dirName: string): Promise { + for (const dir of await getAllDirectories()) { + if (normalizePath(dir.path) === normalizePath(dirName)) { + return dir.id; + } + } + + throw new Error(`Directory '${dirName}' not found in crowdin project`); +} + +async function mkdirAsync(dirName: string): Promise { + const dirs = await getAllDirectories(); + + for (const dir of dirs) { + if (normalizePath(dir.path) === normalizePath(dirName)) { + return dir; + } + } + + let parentDirId: number; + const parentDir = path.dirname(dirName); + if (parentDir && parentDir !== ".") { + parentDirId = (await mkdirAsync(parentDir)).id; + } + + return await createDirectory(path.basename(dirName), parentDirId); +} + +async function getAllDirectories() { + // This request takes a decent amount of time, so cache the results + if (!fetchedDirectories) { + const { sourceFilesApi } = getClient(); + + pxt.debug(`Fetching directories`) + const dirsResponse = await sourceFilesApi + .withFetchAll() + .listProjectDirectories(projectId, {}); + + let dirs = dirsResponse.data.map(fileResponse => fileResponse.data); + + if (!dirs.length) { + throw new Error("No directories found!"); + } + + pxt.debug(`Directory count: ${dirs.length}`); + + fetchedDirectories = dirs; + } + + return fetchedDirectories; +} + +async function getAllFiles() { + // This request takes a decent amount of time, so cache the results + if (!fetchedFiles) { + const { sourceFilesApi } = getClient(); + + pxt.debug(`Fetching files`) + const filesResponse = await sourceFilesApi + .withFetchAll() + .listProjectFiles(projectId, {}); + + let files = filesResponse.data.map(fileResponse => fileResponse.data); + + if (!files.length) { + throw new Error("No files found!"); + } + + pxt.debug(`File count: ${files.length}`); + fetchedFiles = files; + } + + return fetchedFiles; +} + +async function createFile(fileName: string, fileContent: any, directoryId?: number): Promise { + if (pxt.crowdin.testMode) return; + + const { uploadStorageApi, sourceFilesApi } = getClient(); + + // This request happens in two parts: first we upload the file to the storage API, + // then we actually create the file + const storageResponse = await uploadStorageApi.addStorage(fileName, fileContent); + + const file = await sourceFilesApi.createFile(projectId, { + storageId: storageResponse.data.id, + name: fileName, + directoryId + }); + + // Make sure to add the file to the cache if it exists + if (fetchedFiles) { + fetchedFiles.push(file.data); + } +} + +async function createDirectory(dirName: string, directoryId?: number): Promise { + if (pxt.crowdin.testMode) return undefined; + + const { sourceFilesApi } = getClient(); + + const dir = await sourceFilesApi.createDirectory(projectId, { + name: dirName, + directoryId + }); + + // Make sure to add the directory to the cache if it exists + if (fetchedDirectories) { + fetchedDirectories.push(dir.data); + } + return dir.data; +} + +export async function restoreFileBefore(filename: string, cutoffTime: number) { + const revisions = await listFileRevisions(filename); + + let lastRevision: SourceFilesModel.FileRevision; + let lastRevisionBeforeCutoff: SourceFilesModel.FileRevision; + + for (const rev of revisions) { + const time = new Date(rev.date).getTime(); + + if (lastRevision) { + if (time > new Date(lastRevision.date).getTime()) { + lastRevision = rev; + } + } + else { + lastRevision = rev; + } + + if (time < cutoffTime) { + if (lastRevisionBeforeCutoff) { + if (time > new Date(lastRevisionBeforeCutoff.date).getTime()) { + lastRevisionBeforeCutoff = rev; + } + } + else { + lastRevisionBeforeCutoff = rev; + } + } + } + + if (lastRevision === lastRevisionBeforeCutoff) { + pxt.log(`${filename} already at most recent valid revision before ${formatTime(cutoffTime)}`); + } + else if (lastRevisionBeforeCutoff) { + pxt.log(`Restoring ${filename} to revision ${formatTime(new Date(lastRevisionBeforeCutoff.date).getTime())}`) + await restorefile(lastRevisionBeforeCutoff.fileId, lastRevisionBeforeCutoff.id); + } + else { + pxt.log(`No revisions found for ${filename} before ${formatTime(cutoffTime)}`); + } +} + +function formatTime(time: number) { + const date = new Date(time); + return `${date.toLocaleDateString()} ${date.toLocaleTimeString()}`; +} + +async function listFileRevisions(filename: string): Promise { + const { sourceFilesApi } = getClient(); + + const fileId = await getFileIdAsync(filename); + const revisions = await sourceFilesApi + .withFetchAll() + .listFileRevisions(projectId, fileId); + + return revisions.data.map(rev => rev.data); +} + + +async function updateFile(fileId: number, fileName: string, fileContent: any): Promise { + if (pxt.crowdin.testMode) return; + + const { uploadStorageApi, sourceFilesApi } = getClient(); + + const storageResponse = await uploadStorageApi.addStorage(fileName, fileContent); + + await sourceFilesApi.updateOrRestoreFile(projectId, fileId, { + storageId: storageResponse.data.id, + updateOption: "keep_translations" + }); +} + +async function restorefile(fileId: number, revisionId: number) { + if (pxt.crowdin.testMode) return; + + const { sourceFilesApi } = getClient(); + + await sourceFilesApi.updateOrRestoreFile(projectId, fileId, { + revisionId + }); +} + +function getClient() { + if (!client) { + const crowdinConfig: ClientConfig = { + retryConfig: { + retries: 5, + waitInterval: 5000, + conditions: [ + { + test: (error: any) => { + // do not retry when result has not changed + return error?.code == 304; + } + } + ] + } + }; + + client = new crowdin(crowdinCredentials(), crowdinConfig); + } + + return client; +} + +function crowdinCredentials(): Credentials { + const token = process.env[pxt.crowdin.KEY_VARIABLE]; + + if (!token) { + throw new Error(`Crowdin token not found in environment variable ${pxt.crowdin.KEY_VARIABLE}`); + } + + if (pxt.appTarget?.appTheme?.crowdinProjectId !== undefined) { + setProjectId(pxt.appTarget.appTheme.crowdinProjectId); + } + + return { token }; +} + +// calls path.normalize and removes leading slash +function normalizePath(p: string) { + p = path.normalize(p); + p = p.replace(/\\/g, "/"); + if (/^[\/\\]/.test(p)) p = p.slice(1) + + return p; +} \ No newline at end of file diff --git a/localtypings/pxtarget.d.ts b/localtypings/pxtarget.d.ts index ee4495e1a508..82a7fab8414d 100644 --- a/localtypings/pxtarget.d.ts +++ b/localtypings/pxtarget.d.ts @@ -363,6 +363,7 @@ declare namespace pxt { simAnimationExit?: string; // Simulator exit animation hasAudio?: boolean; // target uses the Audio manager. if true: a mute button is added to the simulator toolbar. crowdinProject?: string; + crowdinProjectId?: number; // Crowdin project id. Can be found by going to the project page in Crowdin and selecting Tools > API crowdinBranch?: string; // optional branch specification for localization files monacoToolbox?: boolean; // if true: show the monaco toolbox when in the monaco editor blockHats?: boolean; // if true, event blocks have hats diff --git a/package.json b/package.json index dce610af5ff0..4aa7608a0cb5 100644 --- a/package.json +++ b/package.json @@ -67,10 +67,13 @@ "@blockly/keyboard-navigation": "^0.1.18", "@blockly/plugin-workspace-search": "^4.0.10", "@fortawesome/fontawesome-free": "^5.15.4", + "@crowdin/crowdin-api-client": "^1.33.0", "@microsoft/applicationinsights-web": "^2.8.11", "@microsoft/immersive-reader-sdk": "1.1.0", "@types/diff-match-patch": "^1.0.32", "@zip.js/zip.js": "2.4.20", + "adm-zip": "^0.5.12", + "axios": "^1.6.8", "browserify": "16.2.3", "chai": "^3.5.0", "cssnano": "4.1.10", @@ -103,6 +106,7 @@ }, "devDependencies": { "@microsoft/eslint-plugin-sdl": "^0.1.5", + "@types/adm-zip": "^0.5.5", "@types/chai": "4.0.6", "@types/fuse": "2.5.2", "@types/highlight.js": "9.12.2", diff --git a/pxtlib/crowdin.ts b/pxtlib/crowdin.ts index c9ece9609146..427a20b11172 100644 --- a/pxtlib/crowdin.ts +++ b/pxtlib/crowdin.ts @@ -1,296 +1,12 @@ namespace pxt.crowdin { export const KEY_VARIABLE = "CROWDIN_KEY"; export let testMode = false; - export const TEST_KEY = "!!!testmode!!!"; export function setTestMode() { pxt.crowdin.testMode = true; pxt.log(`CROWDIN TEST MODE - files will NOT be uploaded`); } - function multipartPostAsync(key: string, uri: string, data: any = {}, filename: string = null, filecontents: string = null): Promise { - if (testMode || key == TEST_KEY) { - const resp = { - success: true - } - return Promise.resolve({ statusCode: 200, headers: {}, text: JSON.stringify(resp), json: resp }) - } - return Util.multipartPostAsync(uri, data, filename, filecontents); - } - - function apiUri(branch: string, prj: string, key: string, cmd: string, args?: Map) { - Util.assert(!!prj && !!key && !!cmd); - const apiRoot = "https://api.crowdin.com/api/project/" + prj + "/"; - args = args || {}; - if (testMode) - delete args["key"]; // ensure no key is passed in test mode - else - args["key"] = key; - if (branch) - args["branch"] = branch; - return apiRoot + cmd + "?" + Object.keys(args).map(k => `${k}=${encodeURIComponent(args[k])}`).join("&"); - } - - export interface CrowdinFileInfo { - name: string; - fullName?: string; - id: number; - node_type: "file" | "directory" | "branch"; - phrases?: number; - translated?: number; - approved?: number; - branch?: string; - files?: CrowdinFileInfo[]; - } - - export interface CrowdinProjectInfo { - languages: { name: string; code: string; }[]; - files: CrowdinFileInfo[]; - } - - export interface DownloadOptions { - translatedOnly?: boolean; - validatedOnly?: boolean; - } - - export function downloadTranslationsAsync(branch: string, prj: string, key: string, filename: string, options: DownloadOptions = {}): Promise>> { - const q: Map = { json: "true" } - const infoUri = apiUri(branch, prj, key, "info", q); - - const r: Map> = {}; - filename = normalizeFileName(filename); - return Util.httpGetTextAsync(infoUri).then(respText => { - const info = JSON.parse(respText) as CrowdinProjectInfo; - if (!info) throw new Error("info failed") - - let todo = info.languages.filter(l => l.code != "en"); - if (pxt.appTarget && pxt.appTarget.appTheme && pxt.appTarget.appTheme.availableLocales) - todo = todo.filter(l => pxt.appTarget.appTheme.availableLocales.indexOf(l.code) > -1); - pxt.log('languages: ' + todo.map(l => l.code).join(', ')); - const nextFile = (): Promise => { - const item = todo.pop(); - if (!item) return Promise.resolve(); - const exportFileUri = apiUri(branch, prj, key, "export-file", { - file: filename, - language: item.code, - export_translated_only: options.translatedOnly ? "1" : "0", - export_approved_only: options.validatedOnly ? "1" : "0" - }); - pxt.log(`downloading ${item.name} - ${item.code} (${todo.length} more)`) - return Util.httpGetTextAsync(exportFileUri).then((transationsText) => { - try { - const translations = JSON.parse(transationsText) as Map; - if (translations) - r[item.code] = translations; - } catch (e) { - pxt.log(exportFileUri + ' ' + e) - } - return nextFile(); - }).then(() => Util.delay(1000)); // throttling otherwise crowdin fails - }; - - return nextFile(); - }).then(() => r); - } - - function mkIncr(filename: string): () => void { - let cnt = 0 - return function incr() { - if (cnt++ > 10) { - throw new Error("Too many API calls for " + filename); - } - } - } - - export function createDirectoryAsync(branch: string, prj: string, key: string, name: string, incr?: () => void): Promise { - name = normalizeFileName(name); - pxt.debug(`create directory ${branch || ""}/${name}`) - if (!incr) incr = mkIncr(name); - return multipartPostAsync(key, apiUri(branch, prj, key, "add-directory"), { json: "true", name: name }) - .then(resp => { - pxt.debug(`crowdin resp: ${resp.statusCode}`) - // 400 returned by folder already exists - if (resp.statusCode == 200 || resp.statusCode == 400) - return Promise.resolve(); - - if (resp.statusCode == 500 && resp.text) { - const json = JSON.parse(resp.text); - if (json.error.code === 50) { - pxt.log('directory already exists') - return Promise.resolve(); - } - } - - const data: any = resp.json || JSON.parse(resp.text) || { error: {} } - if (resp.statusCode == 404 && data.error.code == 17) { - pxt.log(`parent directory missing for ${name}`) - const par = name.replace(/\/[^\/]+$/, "") - if (par != name) { - return createDirectoryAsync(branch, prj, key, par, incr) - .then(() => createDirectoryAsync(branch, prj, key, name, incr)); // retry - } - } - - throw new Error(`cannot create directory ${branch || ""}/${name}: ${resp.statusCode} ${JSON.stringify(data)}`) - }) - } - - function normalizeFileName(filename: string): string { - return filename.replace(/\\/g, '/'); - } - - export function uploadTranslationAsync(branch: string, prj: string, key: string, filename: string, data: string) { - Util.assert(!!prj); - Util.assert(!!key); - - filename = normalizeFileName(filename); - const incr = mkIncr(filename); - - function startAsync(): Promise { - return uploadAsync("update-file", { update_option: "update_as_unapproved" }) - } - - function uploadAsync(op: string, opts: any): Promise { - opts["type"] = "auto"; - opts["json"] = ""; - opts["escape_quotes"] = "0"; - incr(); - return multipartPostAsync(key, apiUri(branch, prj, key, op), opts, filename, data) - .then(resp => handleResponseAsync(resp)) - } - - function handleResponseAsync(resp: Util.HttpResponse) { - const code = resp.statusCode; - const errorData: { - success?: boolean; - error?: { - code: number; - } - } = Util.jsonTryParse(resp.text) || {}; - - pxt.debug(`upload result: ${code}`); - if (code == 404 && errorData.error && errorData.error.code == 8) { - pxt.log(`create new translation file: ${filename}`) - return uploadAsync("add-file", {}) - } - else if (code == 404 && errorData.error?.code == 17) { - return createDirectoryAsync(branch, prj, key, filename.replace(/\/[^\/]+$/, ""), incr) - .then(() => startAsync()) - } else if (!errorData.success && errorData.error?.code == 53) { - // file is being updated - pxt.log(`${filename} being updated, waiting 5s and retry...`) - return U.delay(5000) // wait 5s and try again - .then(() => uploadTranslationAsync(branch, prj, key, filename, data)); - } else if (code == 429 && errorData.error?.code == 55) { - // Too many concurrent requests - pxt.log(`Maximum concurrent requests reached, waiting 10s and retry...`) - return U.delay(10 * 1000) // wait 10s and try again - .then(() => uploadTranslationAsync(branch, prj, key, filename, data)); - } else if (code == 200 || errorData.success) { - // something crowdin reports 500 with success=true - return Promise.resolve() - } else { - throw new Error(`Error, upload translation: ${filename}, ${code}, ${resp.text}`) - } - } - - return startAsync(); - } - - function flatten(allFiles: CrowdinFileInfo[], node: CrowdinFileInfo, parentDir: string, branch?: string) { - const n = node.name; - const d = parentDir ? parentDir + "/" + n : n; - node.fullName = d; - node.branch = branch || ""; - switch (node.node_type) { - case "file": - allFiles.push(node); - break; - case "directory": - (node.files || []).forEach(f => flatten(allFiles, f, d, branch)); - break; - case "branch": - (node.files || []).forEach(f => flatten(allFiles, f, parentDir, node.name)); - break; - } - } - - function filterAndFlattenFiles(files: CrowdinFileInfo[], crowdinPath?: string): CrowdinFileInfo[] { - const pxtCrowdinBranch = pxt.appTarget.versions.pxtCrowdinBranch || ""; - const targetCrowdinBranch = pxt.appTarget.versions.targetCrowdinBranch || ""; - - let allFiles: CrowdinFileInfo[] = []; - - // flatten the files - files.forEach(f => flatten(allFiles, f, "")); - - // top level files are for PXT, subolder are targets - allFiles = allFiles.filter(f => { - if (f.fullName.indexOf('/') < 0) return f.branch == pxtCrowdinBranch; // pxt file - else return f.branch == targetCrowdinBranch; - }) - - // folder filter - if (crowdinPath) { - // filter out crowdin folder - allFiles = allFiles.filter(f => f.fullName.indexOf(crowdinPath) == 0); - } - - // filter out non-target files - if (pxt.appTarget.id != "core") { - const id = pxt.appTarget.id + '/' - allFiles = allFiles.filter(f => { - return f.fullName.indexOf('/') < 0 // top level file - || f.fullName.substr(0, id.length) == id // from the target folder - || f.fullName.indexOf('common-docs') >= 0 // common docs - }) - } - - return allFiles; - } - - export function projectInfoAsync(prj: string, key: string): Promise { - const q: Map = { json: "true" } - const infoUri = apiUri("", prj, key, "info", q); - return Util.httpGetTextAsync(infoUri).then(respText => { - const info = JSON.parse(respText) as CrowdinProjectInfo; - return info; - }); - } - - /** - * Scans files in crowdin and report files that are not on disk anymore - */ - export function listFilesAsync(prj: string, key: string, crowdinPath: string): Promise<{ fullName: string; branch: string; }[]> { - - pxt.log(`crowdin: listing files under ${crowdinPath}`); - - return projectInfoAsync(prj, key) - .then(info => { - if (!info) throw new Error("info failed") - - let allFiles = filterAndFlattenFiles(info.files, crowdinPath); - pxt.debug(`crowdin: found ${allFiles.length} under ${crowdinPath}`) - - return allFiles.map(f => { - return { - fullName: f.fullName, - branch: f.branch || "" - }; - }) - }); - } - - export function languageStatsAsync(prj: string, key: string, lang: string): Promise { - const uri = apiUri("", prj, key, "language-status", { language: lang, json: "true" }); - - return Util.httpGetJsonAsync(uri) - .then(info => { - const allFiles = filterAndFlattenFiles(info.files); - return allFiles; - }); - } - export function inContextLoadAsync(text: string): Promise { const node = document.createElement("input") as HTMLInputElement; node.type = "text"; From 65c0b2869561562f18bcc18ba2323232ffdebd64 Mon Sep 17 00:00:00 2001 From: Richard Knoll Date: Fri, 23 Feb 2024 10:47:17 -0800 Subject: [PATCH 2/4] hide asset name when assetEditor is false (#9878) --- pxtlib/package.ts | 15 +++++--- .../src/components/ImageEditor/BottomBar.tsx | 34 +++++++++++-------- .../components/ImageEditor/ImageEditor.tsx | 17 ++++++++-- webapp/src/components/ImageFieldEditor.tsx | 7 +++- webapp/src/components/TilemapFieldEditor.tsx | 2 +- 5 files changed, 52 insertions(+), 23 deletions(-) diff --git a/pxtlib/package.ts b/pxtlib/package.ts index 02b46398f8fd..61eacbb7353c 100644 --- a/pxtlib/package.ts +++ b/pxtlib/package.ts @@ -833,10 +833,8 @@ namespace pxt { await loadDepsRecursive(null, this); // get paletter config loading deps, so the more higher level packages take precedence - if (this.config.palette && appTarget.runtime) { - appTarget.runtime.palette = U.clone(this.config.palette); - if (this.config.paletteNames) appTarget.runtime.paletteNames = this.config.paletteNames; - } + this.patchAppTargetPalette(); + // get screen size loading deps, so the more higher level packages take precedence if (this.config.screenSize && appTarget.runtime) appTarget.runtime.screenSize = U.clone(this.config.screenSize); @@ -997,6 +995,13 @@ namespace pxt { return r; } + + protected patchAppTargetPalette() { + if (this.config.palette && appTarget.runtime) { + appTarget.runtime.palette = U.clone(this.config.palette); + if (this.config.paletteNames) appTarget.runtime.paletteNames = this.config.paletteNames; + } + } } export class MainPackage extends Package { @@ -1288,6 +1293,8 @@ namespace pxt { opts.jres = this.getJRes() const functionOpts = pxt.appTarget.runtime && pxt.appTarget.runtime.functionsOptions; opts.allowedArgumentTypes = functionOpts && functionOpts.extraFunctionEditorTypes && functionOpts.extraFunctionEditorTypes.map(info => info.typeName).concat("number", "boolean", "string"); + + this.patchAppTargetPalette(); return opts; } diff --git a/webapp/src/components/ImageEditor/BottomBar.tsx b/webapp/src/components/ImageEditor/BottomBar.tsx index 9c9b886505a2..8107117e47c1 100644 --- a/webapp/src/components/ImageEditor/BottomBar.tsx +++ b/webapp/src/components/ImageEditor/BottomBar.tsx @@ -22,6 +22,7 @@ export interface BottomBarProps { aspectRatioLocked: boolean; onionSkinEnabled: boolean; + hideAssetName: boolean; dispatchUndoImageEdit: () => void; dispatchRedoImageEdit: () => void; @@ -67,7 +68,8 @@ export class BottomBarImpl extends React.Component
- - {assetNameMessage &&
- {assetNameMessage} -
} + {!hideAssetName && + <> + + {assetNameMessage &&
+ {assetNameMessage} +
} + + }
{isAnimationEditor && !singleFrame ? : undefined}
- + {alert && alert.title && } - {editingTile && } + {editingTile && + + } } diff --git a/webapp/src/components/ImageFieldEditor.tsx b/webapp/src/components/ImageFieldEditor.tsx index e44d9fe5cf29..e1e0adcaa500 100644 --- a/webapp/src/components/ImageFieldEditor.tsx +++ b/webapp/src/components/ImageFieldEditor.tsx @@ -87,13 +87,17 @@ export class ImageFieldEditor extends React.Component extends React.Component }
- +
} From a04fd01048cebf7d391abaf3d2f8a616929bfb1b Mon Sep 17 00:00:00 2001 From: Richard Knoll Date: Mon, 4 Mar 2024 09:23:31 -0800 Subject: [PATCH 3/4] Fixes for image editor in micro:bit (#9900) * fix compile variants when palette is in dep * add field editor param for the tagged template --- pxtblocks/fields/field_animation.ts | 13 +++++++++---- pxtblocks/fields/field_sprite.ts | 9 +++++++-- pxtlib/package.ts | 6 +++++- pxtlib/spriteutils.ts | 13 +++++++------ pxtlib/tsconfig.json | 3 ++- 5 files changed, 30 insertions(+), 14 deletions(-) diff --git a/pxtblocks/fields/field_animation.ts b/pxtblocks/fields/field_animation.ts index c554c8186f7d..d6c804b9830e 100644 --- a/pxtblocks/fields/field_animation.ts +++ b/pxtblocks/fields/field_animation.ts @@ -11,6 +11,8 @@ namespace pxtblockly { filter?: string; lightMode: boolean; + + taggedTemplate?: string; } export interface ParsedFieldAnimationOptions { @@ -19,6 +21,8 @@ namespace pxtblockly { disableResize: boolean; filter?: string; lightMode: boolean; + + taggedTemplate?: string; } // 32 is specifically chosen so that we can scale the images for the default @@ -71,7 +75,7 @@ namespace pxtblockly { const existing = pxt.lookupProjectAssetByTSReference(text, project); if (existing) return existing; - const frames = parseImageArrayString(text); + const frames = parseImageArrayString(text, this.params.taggedTemplate); if (frames && frames.length) { const id = this.sourceBlock_.id; @@ -115,7 +119,7 @@ namespace pxtblockly { if (this.isTemporaryAsset()) { return "[" + this.asset.frames.map(frame => - pxt.sprite.bitmapToImageLiteral(pxt.sprite.Bitmap.fromData(frame), pxt.editor.FileType.TypeScript) + pxt.sprite.bitmapToImageLiteral(pxt.sprite.Bitmap.fromData(frame), pxt.editor.FileType.TypeScript, this.params.taggedTemplate) ).join(",") + "]" } @@ -236,6 +240,7 @@ namespace pxtblockly { parsed.initWidth = withDefault(opts.initWidth, parsed.initWidth); parsed.initHeight = withDefault(opts.initHeight, parsed.initHeight); + parsed.taggedTemplate = opts.taggedTemplate; return parsed; @@ -248,10 +253,10 @@ namespace pxtblockly { } } - function parseImageArrayString(str: string): pxt.sprite.BitmapData[] { + function parseImageArrayString(str: string, templateLiteral?: string): pxt.sprite.BitmapData[] { if (str.indexOf("[") === -1) return null; str = str.replace(/[\[\]]/mg, ""); - return str.split(",").map(s => pxt.sprite.imageLiteralToBitmap(s).data()).filter(b => b.height && b.width); + return str.split(",").map(s => pxt.sprite.imageLiteralToBitmap(s, templateLiteral).data()).filter(b => b.height && b.width); } function isNumberType(type: string) { diff --git a/pxtblocks/fields/field_sprite.ts b/pxtblocks/fields/field_sprite.ts index b063d1b93286..5453e4c69a72 100644 --- a/pxtblocks/fields/field_sprite.ts +++ b/pxtblocks/fields/field_sprite.ts @@ -17,6 +17,8 @@ namespace pxtblockly { filter?: string; lightMode: boolean; + + taggedTemplate?: string; } interface ParsedSpriteEditorOptions { @@ -26,6 +28,7 @@ namespace pxtblockly { disableResize: boolean; filter?: string; lightMode: boolean; + taggedTemplate?: string; } export class FieldSpriteEditor extends FieldAssetEditor { @@ -45,7 +48,7 @@ namespace pxtblockly { return project.lookupAsset(pxt.AssetType.Image, this.getBlockData()); } - const bmp = text ? pxt.sprite.imageLiteralToBitmap(text) : new pxt.sprite.Bitmap(this.params.initWidth, this.params.initHeight); + const bmp = text ? pxt.sprite.imageLiteralToBitmap(text, this.params.taggedTemplate) : new pxt.sprite.Bitmap(this.params.initWidth, this.params.initHeight); if (!bmp) { this.isGreyBlock = true; @@ -72,7 +75,7 @@ namespace pxtblockly { if (this.asset && !this.isTemporaryAsset()) { return pxt.getTSReferenceForAsset(this.asset); } - return pxt.sprite.bitmapToImageLiteral(this.asset && pxt.sprite.Bitmap.fromData((this.asset as pxt.ProjectImage).bitmap), pxt.editor.FileType.TypeScript); + return pxt.sprite.bitmapToImageLiteral(this.asset && pxt.sprite.Bitmap.fromData((this.asset as pxt.ProjectImage).bitmap), pxt.editor.FileType.TypeScript, this.params.taggedTemplate); } protected parseFieldOptions(opts: FieldSpriteEditorOptions): ParsedSpriteEditorOptions { @@ -139,6 +142,8 @@ namespace pxtblockly { parsed.initWidth = withDefault(opts.initWidth, parsed.initWidth); parsed.initHeight = withDefault(opts.initHeight, parsed.initHeight); + parsed.taggedTemplate = opts.taggedTemplate; + return parsed; function withDefault(raw: string, def: number) { diff --git a/pxtlib/package.ts b/pxtlib/package.ts index 61eacbb7353c..0b3639e5b171 100644 --- a/pxtlib/package.ts +++ b/pxtlib/package.ts @@ -996,7 +996,7 @@ namespace pxt { return r; } - protected patchAppTargetPalette() { + patchAppTargetPalette() { if (this.config.palette && appTarget.runtime) { appTarget.runtime.palette = U.clone(this.config.palette); if (this.config.paletteNames) appTarget.runtime.paletteNames = this.config.paletteNames; @@ -1294,6 +1294,10 @@ namespace pxt { const functionOpts = pxt.appTarget.runtime && pxt.appTarget.runtime.functionsOptions; opts.allowedArgumentTypes = functionOpts && functionOpts.extraFunctionEditorTypes && functionOpts.extraFunctionEditorTypes.map(info => info.typeName).concat("number", "boolean", "string"); + for (const dep of this.sortedDeps()) { + dep.patchAppTargetPalette(); + } + this.patchAppTargetPalette(); return opts; } diff --git a/pxtlib/spriteutils.ts b/pxtlib/spriteutils.ts index a9c54d897bb7..0be7956ce072 100644 --- a/pxtlib/spriteutils.ts +++ b/pxtlib/spriteutils.ts @@ -601,11 +601,12 @@ namespace pxt.sprite { return result; } - export function imageLiteralToBitmap(text: string): Bitmap { + export function imageLiteralToBitmap(text: string, templateLiteral = "img"): Bitmap { // Strip the tagged template string business and the whitespace. We don't have to exhaustively // replace encoded characters because the compiler will catch any disallowed characters and throw // an error before the decompilation happens. 96 is backtick and 9 is tab text = text.replace(/[ `]|(?:`)|(?: )|(?:img)/g, "").trim(); + text = text.replaceAll(templateLiteral, ""); text = text.replace(/^["`\(\)]*/, '').replace(/["`\(\)]*$/, ''); text = text.replace(/ /g, "\n"); @@ -733,14 +734,14 @@ namespace pxt.sprite { pxt.sprite.trimTilemapTileset(result); } - function imageLiteralPrologue(fileType: "typescript" | "python"): string { + function imageLiteralPrologue(fileType: "typescript" | "python", templateLiteral = "img"): string { let res = ''; switch (fileType) { case "python": - res = "img(\"\"\""; + res = `${templateLiteral}("""`; break; default: - res = "img`"; + res = `${templateLiteral}\``; break; } return res; @@ -776,10 +777,10 @@ namespace pxt.sprite { return res; } - export function bitmapToImageLiteral(bitmap: Bitmap, fileType: "typescript" | "python"): string { + export function bitmapToImageLiteral(bitmap: Bitmap, fileType: "typescript" | "python", templateLiteral = "img"): string { if (!bitmap || bitmap.height === 0 || bitmap.width === 0) return ""; - let res = imageLiteralPrologue(fileType); + let res = imageLiteralPrologue(fileType, templateLiteral); if (bitmap) { const paddingBetweenPixels = (bitmap.width * bitmap.height > 300) ? "" : " "; diff --git a/pxtlib/tsconfig.json b/pxtlib/tsconfig.json index 3e8468af3afd..25e5cc86b6ba 100644 --- a/pxtlib/tsconfig.json +++ b/pxtlib/tsconfig.json @@ -13,7 +13,8 @@ "dom.iterable", "scripthost", "es2017", - "ES2018.Promise" + "ES2018.Promise", + "ES2021.String" ], "types": [ "highlight.js", From 4cc1d07aab281e02448c347d5141f982001c700b Mon Sep 17 00:00:00 2001 From: Richard Knoll Date: Wed, 8 May 2024 10:01:01 -0700 Subject: [PATCH 4/4] fix help link in monaco flyout (#10008) --- webapp/src/monacoFlyout.tsx | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/webapp/src/monacoFlyout.tsx b/webapp/src/monacoFlyout.tsx index bdf68e448550..308f97cf4072 100644 --- a/webapp/src/monacoFlyout.tsx +++ b/webapp/src/monacoFlyout.tsx @@ -350,7 +350,16 @@ export class MonacoFlyout extends data.Component { + if (pxt.blocks.openHelpUrl) { + pxt.blocks.openHelpUrl(helpUrl); + } + else { + window.open(helpUrl); + } + }; const qName = this.getQName(block) || this.getSnippetName(block); const selected = qName == this.state.selectedBlock; @@ -382,9 +391,11 @@ export class MonacoFlyout extends data.Component{description}
{snippet ? snippet : `${qName}(${params ? params.map(p => `${p.name}`).join(", ") : ""})`} - {helpUrl && - - } + {helpUrl && + + + + }
{params &&
{params.map((p, i) => {